2024-11-10 10:59:00,153 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 10:59:00,165 main DEBUG Took 0.009576 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 10:59:00,165 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 10:59:00,165 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 10:59:00,166 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 10:59:00,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,174 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 10:59:00,185 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,186 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,187 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,187 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,187 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,188 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,188 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,189 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,189 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,190 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,191 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,191 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,192 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,193 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,193 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 10:59:00,195 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,195 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 10:59:00,196 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 10:59:00,197 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 10:59:00,199 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 10:59:00,199 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 10:59:00,200 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 10:59:00,201 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 10:59:00,208 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 10:59:00,211 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 10:59:00,212 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 10:59:00,213 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 10:59:00,213 main DEBUG createAppenders(={Console}) 2024-11-10 10:59:00,214 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-10 10:59:00,214 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 10:59:00,214 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-10 10:59:00,215 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 10:59:00,215 main DEBUG OutputStream closed 2024-11-10 10:59:00,215 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 10:59:00,215 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 10:59:00,215 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-10 10:59:00,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 10:59:00,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 10:59:00,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 10:59:00,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 10:59:00,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 10:59:00,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 10:59:00,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 10:59:00,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 10:59:00,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 10:59:00,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 10:59:00,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 10:59:00,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 10:59:00,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 10:59:00,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 10:59:00,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 10:59:00,290 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 10:59:00,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 10:59:00,291 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 10:59:00,294 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 10:59:00,294 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-10 10:59:00,294 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 10:59:00,295 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-10T10:59:00,592 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a 2024-11-10 10:59:00,595 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 10:59:00,596 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T10:59:00,605 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-11-10T10:59:00,639 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=15, ProcessCount=11, AvailableMemoryMB=10388 2024-11-10T10:59:00,641 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T10:59:00,656 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436, deleteOnExit=true 2024-11-10T10:59:00,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T10:59:00,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/test.cache.data in system properties and HBase conf 2024-11-10T10:59:00,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T10:59:00,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir in system properties and HBase conf 2024-11-10T10:59:00,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T10:59:00,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T10:59:00,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T10:59:00,746 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T10:59:00,840 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T10:59:00,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T10:59:00,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T10:59:00,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T10:59:00,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T10:59:00,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T10:59:00,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T10:59:00,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T10:59:00,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T10:59:00,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T10:59:00,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/nfs.dump.dir in system properties and HBase conf 2024-11-10T10:59:00,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/java.io.tmpdir in system properties and HBase conf 2024-11-10T10:59:00,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T10:59:00,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T10:59:00,849 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T10:59:01,431 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T10:59:01,784 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T10:59:01,873 INFO [Time-limited test {}] log.Log(170): Logging initialized @2443ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T10:59:01,956 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T10:59:02,026 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T10:59:02,050 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T10:59:02,050 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T10:59:02,052 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T10:59:02,068 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T10:59:02,071 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir/,AVAILABLE} 2024-11-10T10:59:02,072 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T10:59:02,295 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/java.io.tmpdir/jetty-localhost-38359-hadoop-hdfs-3_4_1-tests_jar-_-any-11777688081162649338/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T10:59:02,304 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:38359} 2024-11-10T10:59:02,304 INFO [Time-limited test {}] server.Server(415): Started @2875ms 2024-11-10T10:59:02,334 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T10:59:02,715 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T10:59:02,722 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T10:59:02,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T10:59:02,723 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T10:59:02,723 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T10:59:02,724 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir/,AVAILABLE} 2024-11-10T10:59:02,725 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T10:59:02,847 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/java.io.tmpdir/jetty-localhost-40343-hadoop-hdfs-3_4_1-tests_jar-_-any-11821441558060709196/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T10:59:02,848 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:40343} 2024-11-10T10:59:02,848 INFO [Time-limited test {}] server.Server(415): Started @3419ms 2024-11-10T10:59:02,906 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T10:59:03,044 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T10:59:03,050 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T10:59:03,052 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T10:59:03,053 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T10:59:03,053 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T10:59:03,056 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir/,AVAILABLE} 2024-11-10T10:59:03,057 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T10:59:03,217 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/java.io.tmpdir/jetty-localhost-43457-hadoop-hdfs-3_4_1-tests_jar-_-any-51275081224629005/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T10:59:03,217 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:43457} 2024-11-10T10:59:03,218 INFO [Time-limited test {}] server.Server(415): Started @3788ms 2024-11-10T10:59:03,220 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T10:59:03,386 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data4/current/BP-1777068962-172.17.0.2-1731236341535/current, will proceed with Du for space computation calculation, 2024-11-10T10:59:03,386 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data1/current/BP-1777068962-172.17.0.2-1731236341535/current, will proceed with Du for space computation calculation, 2024-11-10T10:59:03,386 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data3/current/BP-1777068962-172.17.0.2-1731236341535/current, will proceed with Du for space computation calculation, 2024-11-10T10:59:03,386 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data2/current/BP-1777068962-172.17.0.2-1731236341535/current, will proceed with Du for space computation calculation, 2024-11-10T10:59:03,455 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T10:59:03,456 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T10:59:03,526 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c567dd358ecc6d8 with lease ID 0x33adb429ac7fbd9f: Processing first storage report for DS-c2becad7-cbc8-4f84-967e-e000eb227623 from datanode DatanodeRegistration(127.0.0.1:34643, datanodeUuid=fbbd0ef3-a418-467f-a3e0-8f3e1a449c95, infoPort=35083, infoSecurePort=0, ipcPort=33371, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535) 2024-11-10T10:59:03,527 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c567dd358ecc6d8 with lease ID 0x33adb429ac7fbd9f: from storage DS-c2becad7-cbc8-4f84-967e-e000eb227623 node DatanodeRegistration(127.0.0.1:34643, datanodeUuid=fbbd0ef3-a418-467f-a3e0-8f3e1a449c95, infoPort=35083, infoSecurePort=0, ipcPort=33371, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-10T10:59:03,527 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x129bcac44d857e07 with lease ID 0x33adb429ac7fbda0: Processing first storage report for DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5 from datanode DatanodeRegistration(127.0.0.1:37243, datanodeUuid=a5330bcf-91a3-47cc-8920-c97ab59b9e9c, infoPort=33839, infoSecurePort=0, ipcPort=44753, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535) 2024-11-10T10:59:03,527 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x129bcac44d857e07 with lease ID 0x33adb429ac7fbda0: from storage DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5 node DatanodeRegistration(127.0.0.1:37243, datanodeUuid=a5330bcf-91a3-47cc-8920-c97ab59b9e9c, infoPort=33839, infoSecurePort=0, ipcPort=44753, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T10:59:03,528 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5c567dd358ecc6d8 with lease ID 0x33adb429ac7fbd9f: Processing first storage report for DS-3149990a-381e-43a0-8b1b-35235b1f4604 from datanode DatanodeRegistration(127.0.0.1:34643, datanodeUuid=fbbd0ef3-a418-467f-a3e0-8f3e1a449c95, infoPort=35083, infoSecurePort=0, ipcPort=33371, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535) 2024-11-10T10:59:03,528 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5c567dd358ecc6d8 with lease ID 0x33adb429ac7fbd9f: from storage DS-3149990a-381e-43a0-8b1b-35235b1f4604 node DatanodeRegistration(127.0.0.1:34643, datanodeUuid=fbbd0ef3-a418-467f-a3e0-8f3e1a449c95, infoPort=35083, infoSecurePort=0, ipcPort=33371, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T10:59:03,528 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x129bcac44d857e07 with lease ID 0x33adb429ac7fbda0: Processing first storage report for DS-48c56e56-8551-4d9f-adff-78444695be1e from datanode DatanodeRegistration(127.0.0.1:37243, datanodeUuid=a5330bcf-91a3-47cc-8920-c97ab59b9e9c, infoPort=33839, infoSecurePort=0, ipcPort=44753, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535) 2024-11-10T10:59:03,528 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x129bcac44d857e07 with lease ID 0x33adb429ac7fbda0: from storage DS-48c56e56-8551-4d9f-adff-78444695be1e node DatanodeRegistration(127.0.0.1:37243, datanodeUuid=a5330bcf-91a3-47cc-8920-c97ab59b9e9c, infoPort=33839, infoSecurePort=0, ipcPort=44753, storageInfo=lv=-57;cid=testClusterID;nsid=588389230;c=1731236341535), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T10:59:03,647 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a 2024-11-10T10:59:03,731 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/zookeeper_0, clientPort=52955, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T10:59:03,741 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52955 2024-11-10T10:59:03,753 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:03,757 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:04,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741825_1001 (size=7) 2024-11-10T10:59:04,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741825_1001 (size=7) 2024-11-10T10:59:04,416 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e with version=8 2024-11-10T10:59:04,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T10:59:04,513 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T10:59:04,762 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T10:59:04,773 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T10:59:04,774 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T10:59:04,778 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T10:59:04,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T10:59:04,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T10:59:04,912 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T10:59:04,972 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T10:59:04,981 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T10:59:04,985 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T10:59:05,011 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 19353 (auto-detected) 2024-11-10T10:59:05,012 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T10:59:05,031 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41247 2024-11-10T10:59:05,054 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41247 connecting to ZooKeeper ensemble=127.0.0.1:52955 2024-11-10T10:59:05,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412470x0, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T10:59:05,090 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41247-0x10101881a2a0000 connected 2024-11-10T10:59:05,122 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:05,125 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:05,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T10:59:05,139 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e, hbase.cluster.distributed=false 2024-11-10T10:59:05,164 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T10:59:05,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41247 2024-11-10T10:59:05,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41247 2024-11-10T10:59:05,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41247 2024-11-10T10:59:05,170 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41247 2024-11-10T10:59:05,171 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41247 2024-11-10T10:59:05,292 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T10:59:05,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T10:59:05,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T10:59:05,294 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T10:59:05,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T10:59:05,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T10:59:05,298 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T10:59:05,300 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T10:59:05,301 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36003 2024-11-10T10:59:05,302 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36003 connecting to ZooKeeper ensemble=127.0.0.1:52955 2024-11-10T10:59:05,303 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:05,307 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:05,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360030x0, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T10:59:05,316 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360030x0, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T10:59:05,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36003-0x10101881a2a0001 connected 2024-11-10T10:59:05,320 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T10:59:05,330 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T10:59:05,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T10:59:05,338 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T10:59:05,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36003 2024-11-10T10:59:05,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36003 2024-11-10T10:59:05,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36003 2024-11-10T10:59:05,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36003 2024-11-10T10:59:05,343 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36003 2024-11-10T10:59:05,360 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:41247 2024-11-10T10:59:05,361 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,41247,1731236344565 2024-11-10T10:59:05,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T10:59:05,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T10:59:05,371 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,41247,1731236344565 2024-11-10T10:59:05,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T10:59:05,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:05,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:05,394 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T10:59:05,395 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,41247,1731236344565 from backup master directory 2024-11-10T10:59:05,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T10:59:05,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,41247,1731236344565 2024-11-10T10:59:05,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T10:59:05,398 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T10:59:05,399 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,41247,1731236344565 2024-11-10T10:59:05,401 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T10:59:05,402 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T10:59:05,462 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase.id] with ID: 60006146-9cb1-4e2a-85be-4d12edbfd158 2024-11-10T10:59:05,463 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/.tmp/hbase.id 2024-11-10T10:59:05,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741826_1002 (size=42) 2024-11-10T10:59:05,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741826_1002 (size=42) 2024-11-10T10:59:05,476 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/.tmp/hbase.id]:[hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase.id] 2024-11-10T10:59:05,520 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:05,525 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T10:59:05,544 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-10T10:59:05,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:05,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741827_1003 (size=196) 2024-11-10T10:59:05,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741827_1003 (size=196) 2024-11-10T10:59:05,583 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T10:59:05,585 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T10:59:05,591 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T10:59:05,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741828_1004 (size=1189) 2024-11-10T10:59:05,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741828_1004 (size=1189) 2024-11-10T10:59:05,643 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store 2024-11-10T10:59:05,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741829_1005 (size=34) 2024-11-10T10:59:05,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741829_1005 (size=34) 2024-11-10T10:59:05,669 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T10:59:05,672 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T10:59:05,673 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T10:59:05,673 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T10:59:05,673 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T10:59:05,675 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T10:59:05,675 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T10:59:05,676 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T10:59:05,677 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236345673Disabling compacts and flushes for region at 1731236345673Disabling writes for close at 1731236345675 (+2 ms)Writing region close event to WAL at 1731236345676 (+1 ms)Closed at 1731236345676 2024-11-10T10:59:05,679 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/.initializing 2024-11-10T10:59:05,680 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/WALs/eccb6f743e07,41247,1731236344565 2024-11-10T10:59:05,702 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C41247%2C1731236344565, suffix=, logDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/WALs/eccb6f743e07,41247,1731236344565, archiveDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/oldWALs, maxLogs=10 2024-11-10T10:59:05,711 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C41247%2C1731236344565.1731236345707 2024-11-10T10:59:05,734 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/WALs/eccb6f743e07,41247,1731236344565/eccb6f743e07%2C41247%2C1731236344565.1731236345707 2024-11-10T10:59:05,743 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083),(127.0.0.1/127.0.0.1:33839:33839)] 2024-11-10T10:59:05,744 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T10:59:05,744 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T10:59:05,748 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,749 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,785 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,813 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T10:59:05,817 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:05,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:05,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T10:59:05,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:05,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T10:59:05,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,829 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T10:59:05,829 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:05,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T10:59:05,830 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T10:59:05,833 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:05,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T10:59:05,835 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,839 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,840 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,845 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,846 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,849 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T10:59:05,853 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T10:59:05,857 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T10:59:05,858 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=834418, jitterRate=0.0610179603099823}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T10:59:05,864 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236345760Initializing all the Stores at 1731236345763 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236345763Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236345764 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236345764Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236345764Cleaning up temporary data from old regions at 1731236345846 (+82 ms)Region opened successfully at 1731236345864 (+18 ms) 2024-11-10T10:59:05,865 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T10:59:05,899 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7820257d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T10:59:05,933 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T10:59:05,945 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T10:59:05,945 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T10:59:05,948 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T10:59:05,950 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T10:59:05,955 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-10T10:59:05,955 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T10:59:05,982 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T10:59:05,993 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T10:59:05,995 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T10:59:05,998 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T10:59:05,999 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T10:59:06,009 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T10:59:06,011 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T10:59:06,015 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T10:59:06,026 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T10:59:06,027 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T10:59:06,032 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T10:59:06,050 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T10:59:06,052 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T10:59:06,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T10:59:06,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T10:59:06,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,059 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,41247,1731236344565, sessionid=0x10101881a2a0000, setting cluster-up flag (Was=false) 2024-11-10T10:59:06,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,079 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T10:59:06,081 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,41247,1731236344565 2024-11-10T10:59:06,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,094 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T10:59:06,096 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,41247,1731236344565 2024-11-10T10:59:06,103 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T10:59:06,148 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(746): ClusterId : 60006146-9cb1-4e2a-85be-4d12edbfd158 2024-11-10T10:59:06,151 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T10:59:06,156 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T10:59:06,156 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T10:59:06,161 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T10:59:06,161 DEBUG [RS:0;eccb6f743e07:36003 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3553fd2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T10:59:06,178 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:36003 2024-11-10T10:59:06,182 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T10:59:06,182 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T10:59:06,182 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T10:59:06,182 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T10:59:06,185 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,41247,1731236344565 with port=36003, startcode=1731236345250 2024-11-10T10:59:06,193 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T10:59:06,199 DEBUG [RS:0;eccb6f743e07:36003 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T10:59:06,203 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T10:59:06,210 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,41247,1731236344565 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T10:59:06,219 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T10:59:06,219 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T10:59:06,220 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T10:59:06,220 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T10:59:06,220 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T10:59:06,220 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,220 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T10:59:06,220 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,223 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236376222 2024-11-10T10:59:06,225 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T10:59:06,226 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T10:59:06,227 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T10:59:06,228 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T10:59:06,229 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T10:59:06,230 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T10:59:06,230 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T10:59:06,230 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T10:59:06,231 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,233 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T10:59:06,234 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T10:59:06,235 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:06,235 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T10:59:06,235 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T10:59:06,239 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T10:59:06,240 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T10:59:06,250 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236346241,5,FailOnTimeoutGroup] 2024-11-10T10:59:06,251 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236346250,5,FailOnTimeoutGroup] 2024-11-10T10:59:06,251 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,251 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T10:59:06,254 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,255 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741831_1007 (size=1321) 2024-11-10T10:59:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741831_1007 (size=1321) 2024-11-10T10:59:06,262 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T10:59:06,263 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e 2024-11-10T10:59:06,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741832_1008 (size=32) 2024-11-10T10:59:06,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741832_1008 (size=32) 2024-11-10T10:59:06,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T10:59:06,281 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50853, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T10:59:06,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T10:59:06,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T10:59:06,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:06,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:06,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T10:59:06,289 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41247 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,291 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T10:59:06,291 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:06,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:06,292 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41247 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T10:59:06,295 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T10:59:06,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:06,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:06,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T10:59:06,300 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T10:59:06,300 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:06,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:06,301 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T10:59:06,302 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740 2024-11-10T10:59:06,303 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740 2024-11-10T10:59:06,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T10:59:06,307 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T10:59:06,308 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T10:59:06,311 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T10:59:06,313 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e 2024-11-10T10:59:06,313 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33151 2024-11-10T10:59:06,313 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T10:59:06,316 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T10:59:06,317 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=840241, jitterRate=0.06842228770256042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T10:59:06,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T10:59:06,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236346281Initializing all the Stores at 1731236346283 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236346283Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236346283Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236346283Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236346283Cleaning up temporary data from old regions at 1731236346307 (+24 ms)Region opened successfully at 1731236346320 (+13 ms) 2024-11-10T10:59:06,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T10:59:06,321 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T10:59:06,321 DEBUG [RS:0;eccb6f743e07:36003 {}] zookeeper.ZKUtil(111): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T10:59:06,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T10:59:06,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T10:59:06,321 WARN [RS:0;eccb6f743e07:36003 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T10:59:06,321 INFO [RS:0;eccb6f743e07:36003 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T10:59:06,321 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,322 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T10:59:06,323 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236346320Disabling compacts and flushes for region at 1731236346320Disabling writes for close at 1731236346321 (+1 ms)Writing region close event to WAL at 1731236346322 (+1 ms)Closed at 1731236346322 2024-11-10T10:59:06,326 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,36003,1731236345250] 2024-11-10T10:59:06,327 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T10:59:06,327 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T10:59:06,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T10:59:06,343 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T10:59:06,346 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T10:59:06,352 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T10:59:06,365 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T10:59:06,370 INFO [RS:0;eccb6f743e07:36003 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T10:59:06,370 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,371 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T10:59:06,377 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T10:59:06,378 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,379 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,379 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,379 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,379 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,379 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,380 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T10:59:06,380 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,380 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,380 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,381 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,381 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,381 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T10:59:06,381 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T10:59:06,381 DEBUG [RS:0;eccb6f743e07:36003 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T10:59:06,382 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,382 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,382 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,382 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,383 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,383 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36003,1731236345250-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T10:59:06,403 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T10:59:06,405 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36003,1731236345250-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,405 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,405 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.Replication(171): eccb6f743e07,36003,1731236345250 started 2024-11-10T10:59:06,424 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:06,425 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,36003,1731236345250, RpcServer on eccb6f743e07/172.17.0.2:36003, sessionid=0x10101881a2a0001 2024-11-10T10:59:06,426 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T10:59:06,426 DEBUG [RS:0;eccb6f743e07:36003 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,426 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,36003,1731236345250' 2024-11-10T10:59:06,426 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T10:59:06,427 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T10:59:06,428 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T10:59:06,428 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T10:59:06,428 DEBUG [RS:0;eccb6f743e07:36003 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,428 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,36003,1731236345250' 2024-11-10T10:59:06,428 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T10:59:06,429 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T10:59:06,430 DEBUG [RS:0;eccb6f743e07:36003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T10:59:06,430 INFO [RS:0;eccb6f743e07:36003 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T10:59:06,430 INFO [RS:0;eccb6f743e07:36003 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T10:59:06,497 WARN [eccb6f743e07:41247 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T10:59:06,539 INFO [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36003%2C1731236345250, suffix=, logDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250, archiveDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs, maxLogs=32 2024-11-10T10:59:06,542 INFO [RS:0;eccb6f743e07:36003 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236346541 2024-11-10T10:59:06,551 INFO [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236346541 2024-11-10T10:59:06,552 DEBUG [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33839:33839),(127.0.0.1/127.0.0.1:35083:35083)] 2024-11-10T10:59:06,749 DEBUG [eccb6f743e07:41247 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T10:59:06,762 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,36003,1731236345250 2024-11-10T10:59:06,768 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,36003,1731236345250, state=OPENING 2024-11-10T10:59:06,773 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T10:59:06,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T10:59:06,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T10:59:06,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T10:59:06,777 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T10:59:06,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,36003,1731236345250}] 2024-11-10T10:59:06,954 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T10:59:06,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38281, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T10:59:06,968 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T10:59:06,969 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T10:59:06,972 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36003%2C1731236345250.meta, suffix=.meta, logDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250, archiveDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs, maxLogs=32 2024-11-10T10:59:06,974 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.meta.1731236346973.meta 2024-11-10T10:59:06,982 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.meta.1731236346973.meta 2024-11-10T10:59:06,983 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33839:33839),(127.0.0.1/127.0.0.1:35083:35083)] 2024-11-10T10:59:06,983 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T10:59:06,985 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T10:59:06,988 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T10:59:06,993 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T10:59:06,998 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T10:59:06,999 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T10:59:07,000 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T10:59:07,000 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T10:59:07,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T10:59:07,005 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T10:59:07,005 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:07,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:07,006 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T10:59:07,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T10:59:07,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:07,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:07,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T10:59:07,010 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T10:59:07,010 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:07,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:07,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T10:59:07,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T10:59:07,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:07,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T10:59:07,013 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T10:59:07,014 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740 2024-11-10T10:59:07,017 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740 2024-11-10T10:59:07,019 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T10:59:07,019 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T10:59:07,020 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T10:59:07,023 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T10:59:07,024 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=874321, jitterRate=0.1117575466632843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T10:59:07,024 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T10:59:07,026 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236347000Writing region info on filesystem at 1731236347001 (+1 ms)Initializing all the Stores at 1731236347003 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236347003Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236347003Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236347003Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236347003Cleaning up temporary data from old regions at 1731236347019 (+16 ms)Running coprocessor post-open hooks at 1731236347025 (+6 ms)Region opened successfully at 1731236347026 (+1 ms) 2024-11-10T10:59:07,033 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236346945 2024-11-10T10:59:07,045 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T10:59:07,046 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T10:59:07,047 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,36003,1731236345250 2024-11-10T10:59:07,050 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,36003,1731236345250, state=OPEN 2024-11-10T10:59:07,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T10:59:07,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T10:59:07,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T10:59:07,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T10:59:07,061 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,36003,1731236345250 2024-11-10T10:59:07,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T10:59:07,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,36003,1731236345250 in 282 msec 2024-11-10T10:59:07,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T10:59:07,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec 2024-11-10T10:59:07,074 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T10:59:07,075 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T10:59:07,095 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T10:59:07,096 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,36003,1731236345250, seqNum=-1] 2024-11-10T10:59:07,117 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T10:59:07,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39217, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T10:59:07,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0070 sec 2024-11-10T10:59:07,141 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236347141, completionTime=-1 2024-11-10T10:59:07,143 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T10:59:07,144 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T10:59:07,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T10:59:07,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236407178 2024-11-10T10:59:07,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236467178 2024-11-10T10:59:07,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 34 msec 2024-11-10T10:59:07,181 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41247,1731236344565-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,181 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41247,1731236344565-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,181 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41247,1731236344565-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,183 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:41247, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,183 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,184 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,190 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T10:59:07,213 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.814sec 2024-11-10T10:59:07,214 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T10:59:07,215 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T10:59:07,216 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T10:59:07,217 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T10:59:07,217 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T10:59:07,218 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41247,1731236344565-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T10:59:07,218 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41247,1731236344565-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T10:59:07,226 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T10:59:07,227 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T10:59:07,227 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41247,1731236344565-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T10:59:07,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b598c24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T10:59:07,261 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T10:59:07,261 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T10:59:07,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,41247,-1 for getting cluster id 2024-11-10T10:59:07,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T10:59:07,276 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '60006146-9cb1-4e2a-85be-4d12edbfd158' 2024-11-10T10:59:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T10:59:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "60006146-9cb1-4e2a-85be-4d12edbfd158" 2024-11-10T10:59:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@252d2f19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T10:59:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,41247,-1] 2024-11-10T10:59:07,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T10:59:07,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T10:59:07,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T10:59:07,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36e15a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T10:59:07,292 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T10:59:07,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,36003,1731236345250, seqNum=-1] 2024-11-10T10:59:07,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T10:59:07,304 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T10:59:07,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,41247,1731236344565 2024-11-10T10:59:07,354 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T10:59:07,365 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T10:59:07,369 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T10:59:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is eccb6f743e07,41247,1731236344565 2024-11-10T10:59:07,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33e047f2 2024-11-10T10:59:07,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T10:59:07,387 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T10:59:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T10:59:07,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T10:59:07,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T10:59:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-10T10:59:07,407 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T10:59:07,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-10T10:59:07,410 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:07,412 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T10:59:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T10:59:07,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741835_1011 (size=389) 2024-11-10T10:59:07,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741835_1011 (size=389) 2024-11-10T10:59:07,464 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 135d63348da74373769abc7152619d94, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e 2024-11-10T10:59:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741836_1012 (size=72) 2024-11-10T10:59:07,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741836_1012 (size=72) 2024-11-10T10:59:07,475 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T10:59:07,475 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 135d63348da74373769abc7152619d94, disabling compactions & flushes 2024-11-10T10:59:07,475 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,475 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,475 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. after waiting 0 ms 2024-11-10T10:59:07,475 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,475 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,475 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 135d63348da74373769abc7152619d94: Waiting for close lock at 1731236347475Disabling compacts and flushes for region at 1731236347475Disabling writes for close at 1731236347475Writing region close event to WAL at 1731236347475Closed at 1731236347475 2024-11-10T10:59:07,478 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T10:59:07,483 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1731236347478"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236347478"}]},"ts":"1731236347478"} 2024-11-10T10:59:07,488 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T10:59:07,490 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T10:59:07,493 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236347490"}]},"ts":"1731236347490"} 2024-11-10T10:59:07,497 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-10T10:59:07,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=135d63348da74373769abc7152619d94, ASSIGN}] 2024-11-10T10:59:07,501 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=135d63348da74373769abc7152619d94, ASSIGN 2024-11-10T10:59:07,503 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=135d63348da74373769abc7152619d94, ASSIGN; state=OFFLINE, location=eccb6f743e07,36003,1731236345250; forceNewPlan=false, retain=false 2024-11-10T10:59:07,655 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=135d63348da74373769abc7152619d94, regionState=OPENING, regionLocation=eccb6f743e07,36003,1731236345250 2024-11-10T10:59:07,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=135d63348da74373769abc7152619d94, ASSIGN because future has completed 2024-11-10T10:59:07,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135d63348da74373769abc7152619d94, server=eccb6f743e07,36003,1731236345250}] 2024-11-10T10:59:07,822 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,822 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 135d63348da74373769abc7152619d94, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.', STARTKEY => '', ENDKEY => ''} 2024-11-10T10:59:07,823 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,823 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T10:59:07,823 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,823 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,826 INFO [StoreOpener-135d63348da74373769abc7152619d94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,828 INFO [StoreOpener-135d63348da74373769abc7152619d94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 135d63348da74373769abc7152619d94 columnFamilyName info 2024-11-10T10:59:07,828 DEBUG [StoreOpener-135d63348da74373769abc7152619d94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T10:59:07,830 INFO [StoreOpener-135d63348da74373769abc7152619d94-1 {}] regionserver.HStore(327): Store=135d63348da74373769abc7152619d94/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T10:59:07,830 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,831 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94 2024-11-10T10:59:07,832 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94 2024-11-10T10:59:07,833 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,833 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,836 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,839 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T10:59:07,840 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 135d63348da74373769abc7152619d94; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=867336, jitterRate=0.10287560522556305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T10:59:07,840 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 135d63348da74373769abc7152619d94 2024-11-10T10:59:07,841 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 135d63348da74373769abc7152619d94: Running coprocessor pre-open hook at 1731236347823Writing region info on filesystem at 1731236347823Initializing all the Stores at 1731236347825 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236347825Cleaning up temporary data from old regions at 1731236347833 (+8 ms)Running coprocessor post-open hooks at 1731236347840 (+7 ms)Region opened successfully at 1731236347841 (+1 ms) 2024-11-10T10:59:07,843 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94., pid=6, masterSystemTime=1731236347815 2024-11-10T10:59:07,847 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,847 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:07,848 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=135d63348da74373769abc7152619d94, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,36003,1731236345250 2024-11-10T10:59:07,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 135d63348da74373769abc7152619d94, server=eccb6f743e07,36003,1731236345250 because future has completed 2024-11-10T10:59:07,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T10:59:07,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 135d63348da74373769abc7152619d94, server=eccb6f743e07,36003,1731236345250 in 194 msec 2024-11-10T10:59:07,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T10:59:07,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=135d63348da74373769abc7152619d94, ASSIGN in 361 msec 2024-11-10T10:59:07,866 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T10:59:07,866 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236347866"}]},"ts":"1731236347866"} 2024-11-10T10:59:07,870 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-10T10:59:07,872 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T10:59:07,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 475 msec 2024-11-10T10:59:12,504 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T10:59:12,551 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T10:59:12,552 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-10T10:59:14,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T10:59:14,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T10:59:14,971 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-10T10:59:14,971 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T10:59:14,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T10:59:14,972 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T10:59:14,972 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T10:59:14,972 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T10:59:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41247 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T10:59:17,478 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-10T10:59:17,481 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-10T10:59:17,487 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-10T10:59:17,488 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T10:59:17,489 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236357488 2024-11-10T10:59:17,497 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:17,497 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:17,497 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:17,498 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:17,498 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:17,498 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236346541 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236357488 2024-11-10T10:59:17,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083),(127.0.0.1/127.0.0.1:33839:33839)] 2024-11-10T10:59:17,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236346541 is not closed yet, will try archiving it next time 2024-11-10T10:59:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741833_1009 (size=451) 2024-11-10T10:59:17,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741833_1009 (size=451) 2024-11-10T10:59:17,503 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236346541 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236346541 2024-11-10T10:59:17,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94., hostname=eccb6f743e07,36003,1731236345250, seqNum=2] 2024-11-10T10:59:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36003 {}] regionserver.HRegion(8855): Flush requested on 135d63348da74373769abc7152619d94 2024-11-10T10:59:29,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 135d63348da74373769abc7152619d94 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T10:59:29,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/b8c16babf6244b4a81458e39ce2985a8 is 1080, key is row0001/info:/1731236357510/Put/seqid=0 2024-11-10T10:59:29,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741838_1014 (size=12509) 2024-11-10T10:59:29,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741838_1014 (size=12509) 2024-11-10T10:59:29,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/b8c16babf6244b4a81458e39ce2985a8 2024-11-10T10:59:29,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/b8c16babf6244b4a81458e39ce2985a8 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8 2024-11-10T10:59:29,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8, entries=7, sequenceid=11, filesize=12.2 K 2024-11-10T10:59:29,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 135d63348da74373769abc7152619d94 in 139ms, sequenceid=11, compaction requested=false 2024-11-10T10:59:29,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 135d63348da74373769abc7152619d94: 2024-11-10T10:59:33,644 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T10:59:37,553 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236377553 2024-11-10T10:59:37,762 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:37,762 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:37,762 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:37,763 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:37,763 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:37,763 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:37,763 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236357488 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236377553 2024-11-10T10:59:37,764 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083),(127.0.0.1/127.0.0.1:33839:33839)] 2024-11-10T10:59:37,764 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236357488 is not closed yet, will try archiving it next time 2024-11-10T10:59:37,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741837_1013 (size=12399) 2024-11-10T10:59:37,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741837_1013 (size=12399) 2024-11-10T10:59:37,968 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:40,172 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:42,377 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:44,581 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:44,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36003 {}] regionserver.HRegion(8855): Flush requested on 135d63348da74373769abc7152619d94 2024-11-10T10:59:44,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 135d63348da74373769abc7152619d94 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T10:59:44,783 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:44,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/7773db0ca4224785aa416dcd4d7cd859 is 1080, key is row0008/info:/1731236371542/Put/seqid=0 2024-11-10T10:59:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741840_1016 (size=12509) 2024-11-10T10:59:44,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741840_1016 (size=12509) 2024-11-10T10:59:44,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/7773db0ca4224785aa416dcd4d7cd859 2024-11-10T10:59:44,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/7773db0ca4224785aa416dcd4d7cd859 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/7773db0ca4224785aa416dcd4d7cd859 2024-11-10T10:59:44,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/7773db0ca4224785aa416dcd4d7cd859, entries=7, sequenceid=21, filesize=12.2 K 2024-11-10T10:59:45,022 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:45,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 135d63348da74373769abc7152619d94 in 441ms, sequenceid=21, compaction requested=false 2024-11-10T10:59:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 135d63348da74373769abc7152619d94: 2024-11-10T10:59:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-11-10T10:59:45,023 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T10:59:45,024 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8 because midkey is the same as first or last row 2024-11-10T10:59:46,786 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:47,230 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T10:59:47,230 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T10:59:48,990 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:48,991 WARN [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:48,992 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C36003%2C1731236345250:(num 1731236377553) roll requested 2024-11-10T10:59:48,993 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236388993 2024-11-10T10:59:49,201 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T10:59:49,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:49,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:49,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:49,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:49,202 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T10:59:49,202 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236377553 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236388993 2024-11-10T10:59:49,203 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33839:33839),(127.0.0.1/127.0.0.1:35083:35083)] 2024-11-10T10:59:49,203 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236377553 is not closed yet, will try archiving it next time 2024-11-10T10:59:49,203 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236357488 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236357488 2024-11-10T10:59:49,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741839_1015 (size=7739) 2024-11-10T10:59:49,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741839_1015 (size=7739) 2024-11-10T10:59:51,194 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK], DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK]] 2024-11-10T10:59:52,823 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 135d63348da74373769abc7152619d94, had cached 0 bytes from a total of 25018 2024-11-10T10:59:53,398 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK], DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK]] 2024-11-10T10:59:55,601 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK], DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK]] 2024-11-10T10:59:57,805 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK], DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK]] 2024-11-10T10:59:59,808 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T10:59:59,808 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236399808 2024-11-10T11:00:03,644 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T11:00:04,817 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK], DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK]] 2024-11-10T11:00:04,819 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK], DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK]] 2024-11-10T11:00:04,819 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C36003%2C1731236345250:(num 1731236399808) roll requested 2024-11-10T11:00:04,819 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:04,820 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:04,820 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:04,820 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:04,820 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:04,820 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236388993 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236399808 2024-11-10T11:00:04,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741841_1017 (size=4753) 2024-11-10T11:00:04,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741841_1017 (size=4753) 2024-11-10T11:00:04,827 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083),(127.0.0.1/127.0.0.1:33839:33839)] 2024-11-10T11:00:04,827 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236388993 is not closed yet, will try archiving it next time 2024-11-10T11:00:04,827 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236404827 2024-11-10T11:00:09,830 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:09,830 WARN [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36003 {}] regionserver.HRegion(8855): Flush requested on 135d63348da74373769abc7152619d94 2024-11-10T11:00:09,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 135d63348da74373769abc7152619d94 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:00:09,835 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:09,835 WARN [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:11,831 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T11:00:14,833 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:14,833 WARN [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:14,833 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:14,833 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:14,833 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:14,834 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:14,834 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:14,834 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236399808 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236404827 2024-11-10T11:00:14,835 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083),(127.0.0.1/127.0.0.1:33839:33839)] 2024-11-10T11:00:14,835 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236399808 is not closed yet, will try archiving it next time 2024-11-10T11:00:14,835 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C36003%2C1731236345250:(num 1731236404827) roll requested 2024-11-10T11:00:14,836 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236414835 2024-11-10T11:00:14,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741842_1018 (size=1569) 2024-11-10T11:00:14,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741842_1018 (size=1569) 2024-11-10T11:00:14,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/1134f0d670af44f7956352751e5b50ec is 1080, key is row0015/info:/1731236386584/Put/seqid=0 2024-11-10T11:00:14,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741844_1020 (size=12509) 2024-11-10T11:00:14,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741844_1020 (size=12509) 2024-11-10T11:00:14,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/1134f0d670af44f7956352751e5b50ec 2024-11-10T11:00:14,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/1134f0d670af44f7956352751e5b50ec as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1134f0d670af44f7956352751e5b50ec 2024-11-10T11:00:14,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1134f0d670af44f7956352751e5b50ec, entries=7, sequenceid=31, filesize=12.2 K 2024-11-10T11:00:19,844 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5005 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:19,844 WARN [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5005 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:19,867 INFO [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:19,867 WARN [FSHLog-0-hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e-prefix:eccb6f743e07,36003,1731236345250 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34643,DS-c2becad7-cbc8-4f84-967e-e000eb227623,DISK], DatanodeInfoWithStorage[127.0.0.1:37243,DS-2c28f7d1-8487-4c81-9ef4-c05ea6694cb5,DISK]] 2024-11-10T11:00:19,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 135d63348da74373769abc7152619d94 in 10037ms, sequenceid=31, compaction requested=true 2024-11-10T11:00:19,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 135d63348da74373769abc7152619d94: 2024-11-10T11:00:19,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,867 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-11-10T11:00:19,867 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:00:19,867 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,868 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8 because midkey is the same as first or last row 2024-11-10T11:00:19,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,868 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236404827 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236414835 2024-11-10T11:00:19,869 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35083:35083),(127.0.0.1/127.0.0.1:33839:33839)] 2024-11-10T11:00:19,869 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236404827 is not closed yet, will try archiving it next time 2024-11-10T11:00:19,869 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C36003%2C1731236345250:(num 1731236414835) roll requested 2024-11-10T11:00:19,869 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236377553 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236377553 2024-11-10T11:00:19,869 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236419869 2024-11-10T11:00:19,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 135d63348da74373769abc7152619d94:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:00:19,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741843_1019 (size=438) 2024-11-10T11:00:19,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741843_1019 (size=438) 2024-11-10T11:00:19,871 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236388993 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236388993 2024-11-10T11:00:19,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:00:19,872 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:00:19,873 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236399808 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236399808 2024-11-10T11:00:19,875 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236404827 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236404827 2024-11-10T11:00:19,876 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:00:19,877 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.HStore(1541): 135d63348da74373769abc7152619d94/info is initiating minor compaction (all files) 2024-11-10T11:00:19,878 INFO [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 135d63348da74373769abc7152619d94/info in TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T11:00:19,878 INFO [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/7773db0ca4224785aa416dcd4d7cd859, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1134f0d670af44f7956352751e5b50ec] into tmpdir=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp, totalSize=36.6 K 2024-11-10T11:00:19,879 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8c16babf6244b4a81458e39ce2985a8, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731236357510 2024-11-10T11:00:19,880 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7773db0ca4224785aa416dcd4d7cd859, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1731236371542 2024-11-10T11:00:19,881 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1134f0d670af44f7956352751e5b50ec, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1731236386584 2024-11-10T11:00:19,882 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,883 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,883 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,883 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,883 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,883 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236414835 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236419869 2024-11-10T11:00:19,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33839:33839),(127.0.0.1/127.0.0.1:35083:35083)] 2024-11-10T11:00:19,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236414835 is not closed yet, will try archiving it next time 2024-11-10T11:00:19,884 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36003%2C1731236345250.1731236419884 2024-11-10T11:00:19,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741845_1021 (size=93) 2024-11-10T11:00:19,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741845_1021 (size=93) 2024-11-10T11:00:19,886 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236414835 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs/eccb6f743e07%2C36003%2C1731236345250.1731236414835 2024-11-10T11:00:19,896 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,896 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,896 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,896 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,896 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:19,897 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236419869 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/WALs/eccb6f743e07,36003,1731236345250/eccb6f743e07%2C36003%2C1731236345250.1731236419884 2024-11-10T11:00:19,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741846_1022 (size=1258) 2024-11-10T11:00:19,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741846_1022 (size=1258) 2024-11-10T11:00:19,901 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33839:33839),(127.0.0.1/127.0.0.1:35083:35083)] 2024-11-10T11:00:19,912 INFO [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 135d63348da74373769abc7152619d94#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:00:19,913 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/8b178546add640d28dc4edc970d39a32 is 1080, key is row0001/info:/1731236357510/Put/seqid=0 2024-11-10T11:00:19,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741848_1024 (size=27710) 2024-11-10T11:00:19,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741848_1024 (size=27710) 2024-11-10T11:00:19,933 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/8b178546add640d28dc4edc970d39a32 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/8b178546add640d28dc4edc970d39a32 2024-11-10T11:00:19,951 INFO [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 135d63348da74373769abc7152619d94/info of 135d63348da74373769abc7152619d94 into 8b178546add640d28dc4edc970d39a32(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:00:19,951 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 135d63348da74373769abc7152619d94: 2024-11-10T11:00:19,954 INFO [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94., storeName=135d63348da74373769abc7152619d94/info, priority=13, startTime=1731236419869; duration=0sec 2024-11-10T11:00:19,954 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T11:00:19,954 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:00:19,955 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/8b178546add640d28dc4edc970d39a32 because midkey is the same as first or last row 2024-11-10T11:00:19,955 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T11:00:19,955 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:00:19,955 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/8b178546add640d28dc4edc970d39a32 because midkey is the same as first or last row 2024-11-10T11:00:19,955 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-10T11:00:19,956 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:00:19,956 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/8b178546add640d28dc4edc970d39a32 because midkey is the same as first or last row 2024-11-10T11:00:19,956 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:00:19,956 DEBUG [RS:0;eccb6f743e07:36003-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 135d63348da74373769abc7152619d94:info 2024-11-10T11:00:31,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36003 {}] regionserver.HRegion(8855): Flush requested on 135d63348da74373769abc7152619d94 2024-11-10T11:00:31,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 135d63348da74373769abc7152619d94 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:00:31,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/cd5c8b2418974b51a7ad2e70a9dadb52 is 1080, key is row0022/info:/1731236419886/Put/seqid=0 2024-11-10T11:00:31,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741849_1025 (size=12509) 2024-11-10T11:00:31,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741849_1025 (size=12509) 2024-11-10T11:00:31,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/cd5c8b2418974b51a7ad2e70a9dadb52 2024-11-10T11:00:31,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/cd5c8b2418974b51a7ad2e70a9dadb52 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/cd5c8b2418974b51a7ad2e70a9dadb52 2024-11-10T11:00:31,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/cd5c8b2418974b51a7ad2e70a9dadb52, entries=7, sequenceid=42, filesize=12.2 K 2024-11-10T11:00:31,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for 135d63348da74373769abc7152619d94 in 36ms, sequenceid=42, compaction requested=false 2024-11-10T11:00:31,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 135d63348da74373769abc7152619d94: 2024-11-10T11:00:31,945 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-11-10T11:00:31,945 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:00:31,945 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/8b178546add640d28dc4edc970d39a32 because midkey is the same as first or last row 2024-11-10T11:00:33,644 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T11:00:37,824 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 135d63348da74373769abc7152619d94, had cached 0 bytes from a total of 40219 2024-11-10T11:00:39,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:00:39,921 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:00:39,921 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:00:39,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:39,926 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:39,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:00:39,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:00:39,927 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1767583037, stopped=false 2024-11-10T11:00:39,927 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,41247,1731236344565 2024-11-10T11:00:39,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:39,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:39,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:39,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:39,930 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:00:39,930 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:00:39,930 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:00:39,930 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:39,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:39,930 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:39,931 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,36003,1731236345250' ***** 2024-11-10T11:00:39,931 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:00:39,931 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:00:39,932 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:00:39,932 INFO [RS:0;eccb6f743e07:36003 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:00:39,932 INFO [RS:0;eccb6f743e07:36003 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:00:39,932 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(3091): Received CLOSE for 135d63348da74373769abc7152619d94 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,36003,1731236345250 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:36003. 2024-11-10T11:00:39,933 DEBUG [RS:0;eccb6f743e07:36003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:00:39,933 DEBUG [RS:0;eccb6f743e07:36003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:00:39,933 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 135d63348da74373769abc7152619d94, disabling compactions & flushes 2024-11-10T11:00:39,933 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T11:00:39,933 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:00:39,933 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T11:00:39,933 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. after waiting 0 ms 2024-11-10T11:00:39,933 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T11:00:39,934 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 135d63348da74373769abc7152619d94 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-10T11:00:39,934 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T11:00:39,934 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1325): Online Regions={135d63348da74373769abc7152619d94=TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T11:00:39,934 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:00:39,934 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:00:39,934 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:00:39,934 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:00:39,934 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:00:39,934 DEBUG [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1351): Waiting on 135d63348da74373769abc7152619d94, 1588230740 2024-11-10T11:00:39,934 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-10T11:00:39,940 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/1f8702070f2c44fca43a7b3653381ef8 is 1080, key is row0029/info:/1731236433911/Put/seqid=0 2024-11-10T11:00:39,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741850_1026 (size=8193) 2024-11-10T11:00:39,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741850_1026 (size=8193) 2024-11-10T11:00:39,947 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/1f8702070f2c44fca43a7b3653381ef8 2024-11-10T11:00:39,957 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/.tmp/info/1f8702070f2c44fca43a7b3653381ef8 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1f8702070f2c44fca43a7b3653381ef8 2024-11-10T11:00:39,958 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/info/f3f08bd3ecf0419ea98031d30c00f5e7 is 195, key is TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94./info:regioninfo/1731236347848/Put/seqid=0 2024-11-10T11:00:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741851_1027 (size=7016) 2024-11-10T11:00:39,966 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1f8702070f2c44fca43a7b3653381ef8, entries=3, sequenceid=48, filesize=8.0 K 2024-11-10T11:00:39,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741851_1027 (size=7016) 2024-11-10T11:00:39,967 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/info/f3f08bd3ecf0419ea98031d30c00f5e7 2024-11-10T11:00:39,968 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 135d63348da74373769abc7152619d94 in 33ms, sequenceid=48, compaction requested=true 2024-11-10T11:00:39,974 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/7773db0ca4224785aa416dcd4d7cd859, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1134f0d670af44f7956352751e5b50ec] to archive 2024-11-10T11:00:39,977 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T11:00:39,980 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/archive/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/b8c16babf6244b4a81458e39ce2985a8 2024-11-10T11:00:39,982 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/7773db0ca4224785aa416dcd4d7cd859 to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/archive/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/7773db0ca4224785aa416dcd4d7cd859 2024-11-10T11:00:39,984 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1134f0d670af44f7956352751e5b50ec to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/archive/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/info/1134f0d670af44f7956352751e5b50ec 2024-11-10T11:00:39,990 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/ns/96b5402cb1674b49a0400fe32aab18cf is 43, key is default/ns:d/1731236347123/Put/seqid=0 2024-11-10T11:00:39,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741852_1028 (size=5153) 2024-11-10T11:00:39,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741852_1028 (size=5153) 2024-11-10T11:00:39,997 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/ns/96b5402cb1674b49a0400fe32aab18cf 2024-11-10T11:00:39,998 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=eccb6f743e07:41247 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-10T11:00:40,003 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [b8c16babf6244b4a81458e39ce2985a8=12509, 7773db0ca4224785aa416dcd4d7cd859=12509, 1134f0d670af44f7956352751e5b50ec=12509] 2024-11-10T11:00:40,010 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/default/TestLogRolling-testSlowSyncLogRolling/135d63348da74373769abc7152619d94/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-11-10T11:00:40,012 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T11:00:40,013 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 135d63348da74373769abc7152619d94: Waiting for close lock at 1731236439933Running coprocessor pre-close hooks at 1731236439933Disabling compacts and flushes for region at 1731236439933Disabling writes for close at 1731236439933Obtaining lock to block concurrent updates at 1731236439934 (+1 ms)Preparing flush snapshotting stores in 135d63348da74373769abc7152619d94 at 1731236439934Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731236439934Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. at 1731236439935 (+1 ms)Flushing 135d63348da74373769abc7152619d94/info: creating writer at 1731236439935Flushing 135d63348da74373769abc7152619d94/info: appending metadata at 1731236439939 (+4 ms)Flushing 135d63348da74373769abc7152619d94/info: closing flushed file at 1731236439939Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ada1615: reopening flushed file at 1731236439956 (+17 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 135d63348da74373769abc7152619d94 in 33ms, sequenceid=48, compaction requested=true at 1731236439968 (+12 ms)Writing region close event to WAL at 1731236440004 (+36 ms)Running coprocessor post-close hooks at 1731236440011 (+7 ms)Closed at 1731236440012 (+1 ms) 2024-11-10T11:00:40,013 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1731236347389.135d63348da74373769abc7152619d94. 2024-11-10T11:00:40,023 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/table/92ba4dbb436f4ba4ab1e17fc336f17f1 is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1731236347866/Put/seqid=0 2024-11-10T11:00:40,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741853_1029 (size=5396) 2024-11-10T11:00:40,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741853_1029 (size=5396) 2024-11-10T11:00:40,032 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/table/92ba4dbb436f4ba4ab1e17fc336f17f1 2024-11-10T11:00:40,041 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/info/f3f08bd3ecf0419ea98031d30c00f5e7 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/info/f3f08bd3ecf0419ea98031d30c00f5e7 2024-11-10T11:00:40,049 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/info/f3f08bd3ecf0419ea98031d30c00f5e7, entries=10, sequenceid=11, filesize=6.9 K 2024-11-10T11:00:40,051 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/ns/96b5402cb1674b49a0400fe32aab18cf as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/ns/96b5402cb1674b49a0400fe32aab18cf 2024-11-10T11:00:40,059 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/ns/96b5402cb1674b49a0400fe32aab18cf, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T11:00:40,060 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/.tmp/table/92ba4dbb436f4ba4ab1e17fc336f17f1 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/table/92ba4dbb436f4ba4ab1e17fc336f17f1 2024-11-10T11:00:40,068 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/table/92ba4dbb436f4ba4ab1e17fc336f17f1, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T11:00:40,069 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 135ms, sequenceid=11, compaction requested=false 2024-11-10T11:00:40,075 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T11:00:40,076 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:00:40,076 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:00:40,076 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236439934Running coprocessor pre-close hooks at 1731236439934Disabling compacts and flushes for region at 1731236439934Disabling writes for close at 1731236439934Obtaining lock to block concurrent updates at 1731236439934Preparing flush snapshotting stores in 1588230740 at 1731236439934Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1731236439935 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731236439936 (+1 ms)Flushing 1588230740/info: creating writer at 1731236439936Flushing 1588230740/info: appending metadata at 1731236439957 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731236439957Flushing 1588230740/ns: creating writer at 1731236439974 (+17 ms)Flushing 1588230740/ns: appending metadata at 1731236439989 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731236439990 (+1 ms)Flushing 1588230740/table: creating writer at 1731236440006 (+16 ms)Flushing 1588230740/table: appending metadata at 1731236440022 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731236440023 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1eb70e: reopening flushed file at 1731236440040 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34dc92fb: reopening flushed file at 1731236440050 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@92c21d1: reopening flushed file at 1731236440059 (+9 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 135ms, sequenceid=11, compaction requested=false at 1731236440069 (+10 ms)Writing region close event to WAL at 1731236440071 (+2 ms)Running coprocessor post-close hooks at 1731236440076 (+5 ms)Closed at 1731236440076 2024-11-10T11:00:40,076 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:00:40,135 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,36003,1731236345250; all regions closed. 2024-11-10T11:00:40,136 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,136 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,137 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,137 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,137 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741834_1010 (size=3066) 2024-11-10T11:00:40,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741834_1010 (size=3066) 2024-11-10T11:00:40,144 DEBUG [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs 2024-11-10T11:00:40,144 INFO [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C36003%2C1731236345250.meta:.meta(num 1731236346973) 2024-11-10T11:00:40,145 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,145 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,145 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,145 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,145 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741847_1023 (size=12695) 2024-11-10T11:00:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741847_1023 (size=12695) 2024-11-10T11:00:40,152 DEBUG [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/oldWALs 2024-11-10T11:00:40,152 INFO [RS:0;eccb6f743e07:36003 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C36003%2C1731236345250:(num 1731236419884) 2024-11-10T11:00:40,152 DEBUG [RS:0;eccb6f743e07:36003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:40,152 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:00:40,153 INFO [RS:0;eccb6f743e07:36003 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:00:40,153 INFO [RS:0;eccb6f743e07:36003 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T11:00:40,153 INFO [RS:0;eccb6f743e07:36003 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:00:40,153 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:00:40,153 INFO [RS:0;eccb6f743e07:36003 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36003 2024-11-10T11:00:40,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,36003,1731236345250 2024-11-10T11:00:40,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:00:40,157 INFO [RS:0;eccb6f743e07:36003 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:00:40,160 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,36003,1731236345250] 2024-11-10T11:00:40,162 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,36003,1731236345250 already deleted, retry=false 2024-11-10T11:00:40,163 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,36003,1731236345250 expired; onlineServers=0 2024-11-10T11:00:40,163 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,41247,1731236344565' ***** 2024-11-10T11:00:40,163 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:00:40,163 INFO [M:0;eccb6f743e07:41247 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:00:40,163 INFO [M:0;eccb6f743e07:41247 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:00:40,163 DEBUG [M:0;eccb6f743e07:41247 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:00:40,163 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:00:40,163 DEBUG [M:0;eccb6f743e07:41247 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:00:40,164 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236346250 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236346250,5,FailOnTimeoutGroup] 2024-11-10T11:00:40,164 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236346241 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236346241,5,FailOnTimeoutGroup] 2024-11-10T11:00:40,164 INFO [M:0;eccb6f743e07:41247 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:00:40,164 INFO [M:0;eccb6f743e07:41247 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:00:40,164 DEBUG [M:0;eccb6f743e07:41247 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:00:40,164 INFO [M:0;eccb6f743e07:41247 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:00:40,164 INFO [M:0;eccb6f743e07:41247 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:00:40,164 INFO [M:0;eccb6f743e07:41247 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:00:40,165 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:00:40,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:00:40,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:40,165 DEBUG [M:0;eccb6f743e07:41247 {}] zookeeper.ZKUtil(347): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:00:40,165 WARN [M:0;eccb6f743e07:41247 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:00:40,166 INFO [M:0;eccb6f743e07:41247 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/.lastflushedseqids 2024-11-10T11:00:40,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741854_1030 (size=130) 2024-11-10T11:00:40,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741854_1030 (size=130) 2024-11-10T11:00:40,179 INFO [M:0;eccb6f743e07:41247 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:00:40,179 INFO [M:0;eccb6f743e07:41247 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:00:40,179 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:00:40,179 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:40,179 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:40,179 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:00:40,179 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:40,180 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.02 KB heapSize=29.20 KB 2024-11-10T11:00:40,197 DEBUG [M:0;eccb6f743e07:41247 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0205d56563d2484faf3795b5f540562c is 82, key is hbase:meta,,1/info:regioninfo/1731236347047/Put/seqid=0 2024-11-10T11:00:40,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741855_1031 (size=5672) 2024-11-10T11:00:40,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741855_1031 (size=5672) 2024-11-10T11:00:40,204 INFO [M:0;eccb6f743e07:41247 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0205d56563d2484faf3795b5f540562c 2024-11-10T11:00:40,230 DEBUG [M:0;eccb6f743e07:41247 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fd26a9cdf81641429162c98b130ccbd1 is 766, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731236347874/Put/seqid=0 2024-11-10T11:00:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741856_1032 (size=6247) 2024-11-10T11:00:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741856_1032 (size=6247) 2024-11-10T11:00:40,238 INFO [M:0;eccb6f743e07:41247 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.42 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fd26a9cdf81641429162c98b130ccbd1 2024-11-10T11:00:40,245 INFO [M:0;eccb6f743e07:41247 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd26a9cdf81641429162c98b130ccbd1 2024-11-10T11:00:40,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:40,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36003-0x10101881a2a0001, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:40,260 INFO [RS:0;eccb6f743e07:36003 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:00:40,261 INFO [RS:0;eccb6f743e07:36003 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,36003,1731236345250; zookeeper connection closed. 2024-11-10T11:00:40,261 DEBUG [M:0;eccb6f743e07:41247 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e053d03cf98d4f8aabe149b1a0f5ef1f is 69, key is eccb6f743e07,36003,1731236345250/rs:state/1731236346295/Put/seqid=0 2024-11-10T11:00:40,261 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2f183409 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2f183409 2024-11-10T11:00:40,261 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T11:00:40,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741857_1033 (size=5156) 2024-11-10T11:00:40,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741857_1033 (size=5156) 2024-11-10T11:00:40,269 INFO [M:0;eccb6f743e07:41247 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e053d03cf98d4f8aabe149b1a0f5ef1f 2024-11-10T11:00:40,293 DEBUG [M:0;eccb6f743e07:41247 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e9f7cb0aaa2d46928e69af633ab3ddd4 is 52, key is load_balancer_on/state:d/1731236347360/Put/seqid=0 2024-11-10T11:00:40,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741858_1034 (size=5056) 2024-11-10T11:00:40,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741858_1034 (size=5056) 2024-11-10T11:00:40,300 INFO [M:0;eccb6f743e07:41247 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e9f7cb0aaa2d46928e69af633ab3ddd4 2024-11-10T11:00:40,309 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0205d56563d2484faf3795b5f540562c as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0205d56563d2484faf3795b5f540562c 2024-11-10T11:00:40,316 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0205d56563d2484faf3795b5f540562c, entries=8, sequenceid=59, filesize=5.5 K 2024-11-10T11:00:40,317 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fd26a9cdf81641429162c98b130ccbd1 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fd26a9cdf81641429162c98b130ccbd1 2024-11-10T11:00:40,324 INFO [M:0;eccb6f743e07:41247 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fd26a9cdf81641429162c98b130ccbd1 2024-11-10T11:00:40,324 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fd26a9cdf81641429162c98b130ccbd1, entries=6, sequenceid=59, filesize=6.1 K 2024-11-10T11:00:40,325 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/e053d03cf98d4f8aabe149b1a0f5ef1f as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e053d03cf98d4f8aabe149b1a0f5ef1f 2024-11-10T11:00:40,332 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/e053d03cf98d4f8aabe149b1a0f5ef1f, entries=1, sequenceid=59, filesize=5.0 K 2024-11-10T11:00:40,334 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/e9f7cb0aaa2d46928e69af633ab3ddd4 as hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e9f7cb0aaa2d46928e69af633ab3ddd4 2024-11-10T11:00:40,343 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/e9f7cb0aaa2d46928e69af633ab3ddd4, entries=1, sequenceid=59, filesize=4.9 K 2024-11-10T11:00:40,345 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=59, compaction requested=false 2024-11-10T11:00:40,346 INFO [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:40,347 DEBUG [M:0;eccb6f743e07:41247 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236440179Disabling compacts and flushes for region at 1731236440179Disabling writes for close at 1731236440179Obtaining lock to block concurrent updates at 1731236440180 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236440180Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23576, getHeapSize=29840, getOffHeapSize=0, getCellsCount=70 at 1731236440180Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236440181 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236440181Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236440197 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236440197Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236440211 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236440229 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236440229Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236440245 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236440260 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236440260Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236440276 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236440292 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236440292Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@558dff86: reopening flushed file at 1731236440307 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37c87a6a: reopening flushed file at 1731236440316 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f0196c6: reopening flushed file at 1731236440324 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dae4bf7: reopening flushed file at 1731236440332 (+8 ms)Finished flush of dataSize ~23.02 KB/23576, heapSize ~29.14 KB/29840, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 165ms, sequenceid=59, compaction requested=false at 1731236440345 (+13 ms)Writing region close event to WAL at 1731236440346 (+1 ms)Closed at 1731236440346 2024-11-10T11:00:40,347 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,348 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,348 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,348 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,348 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:40,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37243 is added to blk_1073741830_1006 (size=27973) 2024-11-10T11:00:40,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34643 is added to blk_1073741830_1006 (size=27973) 2024-11-10T11:00:40,352 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:00:40,352 INFO [M:0;eccb6f743e07:41247 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:00:40,352 INFO [M:0;eccb6f743e07:41247 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41247 2024-11-10T11:00:40,352 INFO [M:0;eccb6f743e07:41247 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:00:40,388 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:00:40,454 INFO [M:0;eccb6f743e07:41247 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:00:40,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:40,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41247-0x10101881a2a0000, quorum=127.0.0.1:52955, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:40,459 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:40,462 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:40,462 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:40,463 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:40,463 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:40,466 WARN [BP-1777068962-172.17.0.2-1731236341535 heartbeating to localhost/127.0.0.1:33151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:40,466 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:40,466 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:40,466 WARN [BP-1777068962-172.17.0.2-1731236341535 heartbeating to localhost/127.0.0.1:33151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1777068962-172.17.0.2-1731236341535 (Datanode Uuid a5330bcf-91a3-47cc-8920-c97ab59b9e9c) service to localhost/127.0.0.1:33151 2024-11-10T11:00:40,468 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data3/current/BP-1777068962-172.17.0.2-1731236341535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:40,468 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data4/current/BP-1777068962-172.17.0.2-1731236341535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:40,469 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:40,471 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:40,471 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:40,471 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:40,471 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:40,472 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:40,473 WARN [BP-1777068962-172.17.0.2-1731236341535 heartbeating to localhost/127.0.0.1:33151 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:40,473 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:40,473 WARN [BP-1777068962-172.17.0.2-1731236341535 heartbeating to localhost/127.0.0.1:33151 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1777068962-172.17.0.2-1731236341535 (Datanode Uuid fbbd0ef3-a418-467f-a3e0-8f3e1a449c95) service to localhost/127.0.0.1:33151 2024-11-10T11:00:40,473 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:40,474 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data1/current/BP-1777068962-172.17.0.2-1731236341535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:40,474 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/cluster_5f3e8f19-aae1-3212-ee04-fc8be0397436/data/data2/current/BP-1777068962-172.17.0.2-1731236341535 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:40,475 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:40,486 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:00:40,487 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:40,487 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:40,487 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:40,487 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:40,499 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:00:40,534 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:00:40,548 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=77 (was 12) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33151 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/eccb6f743e07:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@373a7f2c java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:33151 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33151 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:33151 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: master/eccb6f743e07:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/eccb6f743e07:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:33151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:33151 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=36 (was 15) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9919 (was 10388) 2024-11-10T11:00:40,558 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=78, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=36, ProcessCount=11, AvailableMemoryMB=9916 2024-11-10T11:00:40,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T11:00:40,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.log.dir so I do NOT create it in target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376 2024-11-10T11:00:40,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/c93aa989-b70c-5dc7-b832-a35cd6cfc20a/hadoop.tmp.dir so I do NOT create it in target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376 2024-11-10T11:00:40,559 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48, deleteOnExit=true 2024-11-10T11:00:40,559 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/test.cache.data in system properties and HBase conf 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir in system properties and HBase conf 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T11:00:40,560 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T11:00:40,560 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/nfs.dump.dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/java.io.tmpdir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:00:40,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T11:00:40,562 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T11:00:40,577 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:00:40,658 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:40,665 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:40,666 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:40,666 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:40,666 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:00:40,667 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:40,667 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46d26a79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:40,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@59505eb5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:40,796 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1edca743{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/java.io.tmpdir/jetty-localhost-36199-hadoop-hdfs-3_4_1-tests_jar-_-any-12683482953234031046/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:00:40,797 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c443180{HTTP/1.1, (http/1.1)}{localhost:36199} 2024-11-10T11:00:40,797 INFO [Time-limited test {}] server.Server(415): Started @101368ms 2024-11-10T11:00:40,812 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:00:40,885 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:40,888 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:40,889 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:40,889 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:40,889 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:00:40,890 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@69a0f3c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:40,890 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@469dec96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:41,011 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a15ed6a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/java.io.tmpdir/jetty-localhost-36811-hadoop-hdfs-3_4_1-tests_jar-_-any-1510264966994622163/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:41,011 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@b5aedfa{HTTP/1.1, (http/1.1)}{localhost:36811} 2024-11-10T11:00:41,011 INFO [Time-limited test {}] server.Server(415): Started @101582ms 2024-11-10T11:00:41,013 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:41,052 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:41,057 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:41,058 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:41,058 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:41,058 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:00:41,058 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@194f043a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:41,059 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@274298f3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:41,116 WARN [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data2/current/BP-2087770100-172.17.0.2-1731236440597/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:41,116 WARN [Thread-437 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data1/current/BP-2087770100-172.17.0.2-1731236440597/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:41,136 WARN [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e38b145a66380c0 with lease ID 0xd723d106570cb79d: Processing first storage report for DS-498a0f8a-9f2a-4fc0-9aa5-0db9e90ac36d from datanode DatanodeRegistration(127.0.0.1:45191, datanodeUuid=1800d232-93fb-4b26-b5a0-7cfc586a9719, infoPort=37071, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597) 2024-11-10T11:00:41,139 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e38b145a66380c0 with lease ID 0xd723d106570cb79d: from storage DS-498a0f8a-9f2a-4fc0-9aa5-0db9e90ac36d node DatanodeRegistration(127.0.0.1:45191, datanodeUuid=1800d232-93fb-4b26-b5a0-7cfc586a9719, infoPort=37071, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:41,140 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1e38b145a66380c0 with lease ID 0xd723d106570cb79d: Processing first storage report for DS-3ccd572e-74d9-4e23-b7a7-c60d97d6607e from datanode DatanodeRegistration(127.0.0.1:45191, datanodeUuid=1800d232-93fb-4b26-b5a0-7cfc586a9719, infoPort=37071, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597) 2024-11-10T11:00:41,140 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1e38b145a66380c0 with lease ID 0xd723d106570cb79d: from storage DS-3ccd572e-74d9-4e23-b7a7-c60d97d6607e node DatanodeRegistration(127.0.0.1:45191, datanodeUuid=1800d232-93fb-4b26-b5a0-7cfc586a9719, infoPort=37071, infoSecurePort=0, ipcPort=45065, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T11:00:41,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@18492d7d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/java.io.tmpdir/jetty-localhost-41075-hadoop-hdfs-3_4_1-tests_jar-_-any-13442323185942741939/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:41,183 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30ebe7e3{HTTP/1.1, (http/1.1)}{localhost:41075} 2024-11-10T11:00:41,183 INFO [Time-limited test {}] server.Server(415): Started @101754ms 2024-11-10T11:00:41,185 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:41,284 WARN [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data3/current/BP-2087770100-172.17.0.2-1731236440597/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:41,284 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data4/current/BP-2087770100-172.17.0.2-1731236440597/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:41,301 WARN [Thread-452 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64dd6772971c1685 with lease ID 0xd723d106570cb79e: Processing first storage report for DS-5c1d4d10-e776-4ff3-b973-521bdeaa3aa0 from datanode DatanodeRegistration(127.0.0.1:45447, datanodeUuid=516eb0bb-9696-4ae7-976c-54ac0ff2d780, infoPort=41013, infoSecurePort=0, ipcPort=44513, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597) 2024-11-10T11:00:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64dd6772971c1685 with lease ID 0xd723d106570cb79e: from storage DS-5c1d4d10-e776-4ff3-b973-521bdeaa3aa0 node DatanodeRegistration(127.0.0.1:45447, datanodeUuid=516eb0bb-9696-4ae7-976c-54ac0ff2d780, infoPort=41013, infoSecurePort=0, ipcPort=44513, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x64dd6772971c1685 with lease ID 0xd723d106570cb79e: Processing first storage report for DS-25a68fb4-c007-4f09-a094-a47f6278735e from datanode DatanodeRegistration(127.0.0.1:45447, datanodeUuid=516eb0bb-9696-4ae7-976c-54ac0ff2d780, infoPort=41013, infoSecurePort=0, ipcPort=44513, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597) 2024-11-10T11:00:41,304 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x64dd6772971c1685 with lease ID 0xd723d106570cb79e: from storage DS-25a68fb4-c007-4f09-a094-a47f6278735e node DatanodeRegistration(127.0.0.1:45447, datanodeUuid=516eb0bb-9696-4ae7-976c-54ac0ff2d780, infoPort=41013, infoSecurePort=0, ipcPort=44513, storageInfo=lv=-57;cid=testClusterID;nsid=1597610699;c=1731236440597), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:41,314 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376 2024-11-10T11:00:41,316 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/zookeeper_0, clientPort=61774, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T11:00:41,317 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61774 2024-11-10T11:00:41,318 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,319 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:00:41,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:00:41,332 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e with version=8 2024-11-10T11:00:41,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T11:00:41,335 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T11:00:41,335 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:00:41,336 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40551 2024-11-10T11:00:41,338 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40551 connecting to ZooKeeper ensemble=127.0.0.1:61774 2024-11-10T11:00:41,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405510x0, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:00:41,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40551-0x101018997730000 connected 2024-11-10T11:00:41,364 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,366 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:41,369 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e, hbase.cluster.distributed=false 2024-11-10T11:00:41,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:00:41,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40551 2024-11-10T11:00:41,371 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40551 2024-11-10T11:00:41,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40551 2024-11-10T11:00:41,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40551 2024-11-10T11:00:41,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40551 2024-11-10T11:00:41,389 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:00:41,389 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:41,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:41,390 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:00:41,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:41,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:00:41,390 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:00:41,390 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:00:41,391 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36361 2024-11-10T11:00:41,392 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36361 connecting to ZooKeeper ensemble=127.0.0.1:61774 2024-11-10T11:00:41,392 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,395 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363610x0, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:00:41,402 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36361-0x101018997730001 connected 2024-11-10T11:00:41,402 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:41,402 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:00:41,403 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:00:41,404 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T11:00:41,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:00:41,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36361 2024-11-10T11:00:41,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36361 2024-11-10T11:00:41,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36361 2024-11-10T11:00:41,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36361 2024-11-10T11:00:41,412 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36361 2024-11-10T11:00:41,426 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:40551 2024-11-10T11:00:41,426 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:41,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:41,429 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T11:00:41,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,431 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:00:41,432 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,40551,1731236441334 from backup master directory 2024-11-10T11:00:41,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:41,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:41,433 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:00:41,433 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,440 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/hbase.id] with ID: 47d63a03-4238-40c6-94a0-d7a9d70b38a1 2024-11-10T11:00:41,440 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/.tmp/hbase.id 2024-11-10T11:00:41,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:00:41,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:00:41,448 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/.tmp/hbase.id]:[hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/hbase.id] 2024-11-10T11:00:41,465 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:41,465 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T11:00:41,466 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T11:00:41,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:00:41,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:00:41,479 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:00:41,480 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T11:00:41,481 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:00:41,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:00:41,489 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store 2024-11-10T11:00:41,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:00:41,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:00:41,496 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:41,496 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:00:41,496 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:41,497 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:41,497 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:00:41,497 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:41,497 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:41,497 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236441496Disabling compacts and flushes for region at 1731236441496Disabling writes for close at 1731236441497 (+1 ms)Writing region close event to WAL at 1731236441497Closed at 1731236441497 2024-11-10T11:00:41,498 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/.initializing 2024-11-10T11:00:41,498 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/WALs/eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,501 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C40551%2C1731236441334, suffix=, logDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/WALs/eccb6f743e07,40551,1731236441334, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/oldWALs, maxLogs=10 2024-11-10T11:00:41,501 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C40551%2C1731236441334.1731236441501 2024-11-10T11:00:41,506 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/WALs/eccb6f743e07,40551,1731236441334/eccb6f743e07%2C40551%2C1731236441334.1731236441501 2024-11-10T11:00:41,507 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37071:37071),(127.0.0.1/127.0.0.1:41013:41013)] 2024-11-10T11:00:41,508 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:00:41,508 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:41,508 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,508 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,512 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,514 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T11:00:41,515 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:41,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T11:00:41,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:41,517 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T11:00:41,520 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:41,521 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,522 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T11:00:41,522 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:41,523 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,524 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,524 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,526 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,526 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,526 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T11:00:41,528 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:41,530 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:00:41,531 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=813541, jitterRate=0.03447163105010986}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T11:00:41,532 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236441508Initializing all the Stores at 1731236441509 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236441509Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236441512 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236441512Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236441512Cleaning up temporary data from old regions at 1731236441526 (+14 ms)Region opened successfully at 1731236441532 (+6 ms) 2024-11-10T11:00:41,532 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T11:00:41,536 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@389cc1e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:00:41,538 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T11:00:41,538 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T11:00:41,538 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T11:00:41,538 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T11:00:41,539 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T11:00:41,539 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T11:00:41,539 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T11:00:41,542 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T11:00:41,543 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T11:00:41,544 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T11:00:41,545 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T11:00:41,545 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T11:00:41,547 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T11:00:41,547 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T11:00:41,548 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T11:00:41,549 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T11:00:41,550 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T11:00:41,553 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T11:00:41,555 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T11:00:41,556 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T11:00:41,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:41,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:41,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,558 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,40551,1731236441334, sessionid=0x101018997730000, setting cluster-up flag (Was=false) 2024-11-10T11:00:41,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,567 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T11:00:41,568 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:41,579 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T11:00:41,580 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,40551,1731236441334 2024-11-10T11:00:41,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T11:00:41,583 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:41,583 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T11:00:41,584 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T11:00:41,584 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,40551,1731236441334 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T11:00:41,585 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:41,585 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:41,585 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:41,585 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:41,585 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T11:00:41,586 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,586 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:00:41,586 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236471587 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T11:00:41,587 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:41,587 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,588 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T11:00:41,588 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T11:00:41,588 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T11:00:41,588 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T11:00:41,588 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T11:00:41,589 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T11:00:41,589 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,589 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T11:00:41,590 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236441589,5,FailOnTimeoutGroup] 2024-11-10T11:00:41,590 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236441590,5,FailOnTimeoutGroup] 2024-11-10T11:00:41,590 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,590 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T11:00:41,590 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,590 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:00:41,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:00:41,600 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T11:00:41,600 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e 2024-11-10T11:00:41,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:00:41,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:00:41,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:41,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:00:41,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:00:41,614 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:41,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:00:41,614 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(746): ClusterId : 47d63a03-4238-40c6-94a0-d7a9d70b38a1 2024-11-10T11:00:41,615 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:00:41,616 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:00:41,616 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:41,617 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:00:41,617 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:00:41,617 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:00:41,619 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:00:41,619 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,619 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:00:41,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:41,620 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:00:41,620 DEBUG [RS:0;eccb6f743e07:36361 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc8daf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:00:41,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:00:41,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:41,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:41,622 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:00:41,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740 2024-11-10T11:00:41,623 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740 2024-11-10T11:00:41,625 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:00:41,625 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:00:41,626 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:00:41,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:00:41,630 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:00:41,630 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=740526, jitterRate=-0.05837307870388031}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:00:41,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236441610Initializing all the Stores at 1731236441611 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236441611Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236441612 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236441612Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236441612Cleaning up temporary data from old regions at 1731236441625 (+13 ms)Region opened successfully at 1731236441632 (+7 ms) 2024-11-10T11:00:41,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:00:41,632 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:00:41,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:00:41,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:00:41,632 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:00:41,633 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:00:41,633 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236441632Disabling compacts and flushes for region at 1731236441632Disabling writes for close at 1731236441632Writing region close event to WAL at 1731236441633 (+1 ms)Closed at 1731236441633 2024-11-10T11:00:41,634 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:36361 2024-11-10T11:00:41,634 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:00:41,634 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:00:41,634 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:00:41,634 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:41,635 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T11:00:41,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T11:00:41,635 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,40551,1731236441334 with port=36361, startcode=1731236441389 2024-11-10T11:00:41,635 DEBUG [RS:0;eccb6f743e07:36361 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:00:41,637 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:00:41,639 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T11:00:41,639 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41563, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:00:41,639 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40551 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,36361,1731236441389 2024-11-10T11:00:41,640 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40551 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,36361,1731236441389 2024-11-10T11:00:41,642 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e 2024-11-10T11:00:41,642 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38271 2024-11-10T11:00:41,642 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:00:41,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:00:41,644 DEBUG [RS:0;eccb6f743e07:36361 {}] zookeeper.ZKUtil(111): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,36361,1731236441389 2024-11-10T11:00:41,644 WARN [RS:0;eccb6f743e07:36361 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:00:41,644 INFO [RS:0;eccb6f743e07:36361 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:41,645 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/WALs/eccb6f743e07,36361,1731236441389 2024-11-10T11:00:41,645 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,36361,1731236441389] 2024-11-10T11:00:41,649 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:00:41,654 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:00:41,654 INFO [RS:0;eccb6f743e07:36361 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:00:41,654 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,656 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:00:41,657 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:00:41,657 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,658 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,659 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,659 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:41,659 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:00:41,659 DEBUG [RS:0;eccb6f743e07:36361 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:00:41,661 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,661 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,661 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,661 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,661 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,661 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36361,1731236441389-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:00:41,677 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:00:41,677 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36361,1731236441389-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,677 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,677 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.Replication(171): eccb6f743e07,36361,1731236441389 started 2024-11-10T11:00:41,692 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:41,692 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,36361,1731236441389, RpcServer on eccb6f743e07/172.17.0.2:36361, sessionid=0x101018997730001 2024-11-10T11:00:41,692 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:00:41,692 DEBUG [RS:0;eccb6f743e07:36361 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,36361,1731236441389 2024-11-10T11:00:41,693 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,36361,1731236441389' 2024-11-10T11:00:41,693 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:00:41,693 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:00:41,694 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:00:41,694 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:00:41,694 DEBUG [RS:0;eccb6f743e07:36361 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,36361,1731236441389 2024-11-10T11:00:41,694 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,36361,1731236441389' 2024-11-10T11:00:41,694 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:00:41,694 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:00:41,695 DEBUG [RS:0;eccb6f743e07:36361 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:00:41,695 INFO [RS:0;eccb6f743e07:36361 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:00:41,695 INFO [RS:0;eccb6f743e07:36361 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:00:41,789 WARN [eccb6f743e07:40551 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T11:00:41,798 INFO [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36361%2C1731236441389, suffix=, logDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/WALs/eccb6f743e07,36361,1731236441389, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/oldWALs, maxLogs=32 2024-11-10T11:00:41,800 INFO [RS:0;eccb6f743e07:36361 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36361%2C1731236441389.1731236441800 2024-11-10T11:00:41,807 INFO [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/WALs/eccb6f743e07,36361,1731236441389/eccb6f743e07%2C36361%2C1731236441389.1731236441800 2024-11-10T11:00:41,810 DEBUG [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37071:37071),(127.0.0.1/127.0.0.1:41013:41013)] 2024-11-10T11:00:42,039 DEBUG [eccb6f743e07:40551 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T11:00:42,040 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,36361,1731236441389 2024-11-10T11:00:42,042 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,36361,1731236441389, state=OPENING 2024-11-10T11:00:42,046 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T11:00:42,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:42,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:42,048 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:00:42,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:42,048 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:42,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,36361,1731236441389}] 2024-11-10T11:00:42,202 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T11:00:42,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46543, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T11:00:42,209 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T11:00:42,209 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:42,211 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36361%2C1731236441389.meta, suffix=.meta, logDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/WALs/eccb6f743e07,36361,1731236441389, archiveDir=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/oldWALs, maxLogs=32 2024-11-10T11:00:42,213 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36361%2C1731236441389.meta.1731236442213.meta 2024-11-10T11:00:42,219 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/WALs/eccb6f743e07,36361,1731236441389/eccb6f743e07%2C36361%2C1731236441389.meta.1731236442213.meta 2024-11-10T11:00:42,220 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37071:37071),(127.0.0.1/127.0.0.1:41013:41013)] 2024-11-10T11:00:42,220 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:00:42,221 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T11:00:42,221 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T11:00:42,221 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T11:00:42,221 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T11:00:42,221 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:42,221 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T11:00:42,221 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T11:00:42,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:00:42,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:00:42,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:42,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:42,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:00:42,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:00:42,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:42,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:42,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:00:42,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:00:42,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:42,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:42,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:00:42,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:00:42,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:42,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:42,229 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:00:42,230 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740 2024-11-10T11:00:42,231 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740 2024-11-10T11:00:42,233 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:00:42,233 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:00:42,234 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:00:42,235 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:00:42,236 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=745519, jitterRate=-0.052023738622665405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:00:42,236 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T11:00:42,237 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236442221Writing region info on filesystem at 1731236442221Initializing all the Stores at 1731236442222 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236442222Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236442223 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236442223Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236442223Cleaning up temporary data from old regions at 1731236442233 (+10 ms)Running coprocessor post-open hooks at 1731236442236 (+3 ms)Region opened successfully at 1731236442237 (+1 ms) 2024-11-10T11:00:42,238 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236442201 2024-11-10T11:00:42,241 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T11:00:42,241 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T11:00:42,242 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,36361,1731236441389 2024-11-10T11:00:42,243 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,36361,1731236441389, state=OPEN 2024-11-10T11:00:42,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:00:42,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:00:42,252 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,36361,1731236441389 2024-11-10T11:00:42,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:42,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:42,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T11:00:42,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,36361,1731236441389 in 204 msec 2024-11-10T11:00:42,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T11:00:42,259 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 621 msec 2024-11-10T11:00:42,260 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:42,260 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T11:00:42,261 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:00:42,261 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,36361,1731236441389, seqNum=-1] 2024-11-10T11:00:42,262 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:00:42,263 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48745, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:00:42,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 687 msec 2024-11-10T11:00:42,270 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236442270, completionTime=-1 2024-11-10T11:00:42,270 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T11:00:42,270 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T11:00:42,272 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T11:00:42,272 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236502272 2024-11-10T11:00:42,272 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236562272 2024-11-10T11:00:42,272 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 2 msec 2024-11-10T11:00:42,273 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40551,1731236441334-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,273 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40551,1731236441334-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,273 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40551,1731236441334-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,273 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:40551, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,273 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,273 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,275 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.844sec 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40551,1731236441334-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:00:42,278 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40551,1731236441334-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T11:00:42,281 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T11:00:42,281 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T11:00:42,281 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40551,1731236441334-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:42,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5835fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:00:42,315 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,40551,-1 for getting cluster id 2024-11-10T11:00:42,315 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T11:00:42,317 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '47d63a03-4238-40c6-94a0-d7a9d70b38a1' 2024-11-10T11:00:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T11:00:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "47d63a03-4238-40c6-94a0-d7a9d70b38a1" 2024-11-10T11:00:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b0cd008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:00:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,40551,-1] 2024-11-10T11:00:42,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T11:00:42,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:42,321 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T11:00:42,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@205bfd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:00:42,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:00:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,36361,1731236441389, seqNum=-1] 2024-11-10T11:00:42,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:00:42,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:00:42,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,40551,1731236441334 2024-11-10T11:00:42,328 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:42,331 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T11:00:42,331 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:00:42,331 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:00:42,332 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:00:42,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:42,332 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:42,332 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:00:42,332 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:00:42,332 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1771574582, stopped=false 2024-11-10T11:00:42,332 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,40551,1731236441334 2024-11-10T11:00:42,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:42,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:42,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:42,335 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:00:42,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:42,335 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:00:42,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:00:42,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:42,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:42,336 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:42,336 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,36361,1731236441389' ***** 2024-11-10T11:00:42,336 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:00:42,336 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:00:42,337 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,36361,1731236441389 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:36361. 2024-11-10T11:00:42,337 DEBUG [RS:0;eccb6f743e07:36361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:00:42,337 DEBUG [RS:0;eccb6f743e07:36361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:00:42,337 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:00:42,338 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T11:00:42,338 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T11:00:42,338 DEBUG [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T11:00:42,338 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:00:42,338 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:00:42,338 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:00:42,338 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:00:42,338 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:00:42,338 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-10T11:00:42,357 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/.tmp/ns/bc258e34a1be45e48b6371fbe6c8b1d9 is 43, key is default/ns:d/1731236442264/Put/seqid=0 2024-11-10T11:00:42,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741835_1011 (size=5153) 2024-11-10T11:00:42,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741835_1011 (size=5153) 2024-11-10T11:00:42,364 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/.tmp/ns/bc258e34a1be45e48b6371fbe6c8b1d9 2024-11-10T11:00:42,372 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/.tmp/ns/bc258e34a1be45e48b6371fbe6c8b1d9 as hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/ns/bc258e34a1be45e48b6371fbe6c8b1d9 2024-11-10T11:00:42,379 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/ns/bc258e34a1be45e48b6371fbe6c8b1d9, entries=2, sequenceid=6, filesize=5.0 K 2024-11-10T11:00:42,381 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=6, compaction requested=false 2024-11-10T11:00:42,385 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T11:00:42,386 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:00:42,386 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:00:42,386 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236442338Running coprocessor pre-close hooks at 1731236442338Disabling compacts and flushes for region at 1731236442338Disabling writes for close at 1731236442338Obtaining lock to block concurrent updates at 1731236442338Preparing flush snapshotting stores in 1588230740 at 1731236442338Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731236442339 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731236442340 (+1 ms)Flushing 1588230740/ns: creating writer at 1731236442340Flushing 1588230740/ns: appending metadata at 1731236442357 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731236442357Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51cbd952: reopening flushed file at 1731236442371 (+14 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 43ms, sequenceid=6, compaction requested=false at 1731236442381 (+10 ms)Writing region close event to WAL at 1731236442382 (+1 ms)Running coprocessor post-close hooks at 1731236442386 (+4 ms)Closed at 1731236442386 2024-11-10T11:00:42,387 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:00:42,538 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,36361,1731236441389; all regions closed. 2024-11-10T11:00:42,539 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,539 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,539 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,539 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,539 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741834_1010 (size=1152) 2024-11-10T11:00:42,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741834_1010 (size=1152) 2024-11-10T11:00:42,545 DEBUG [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/oldWALs 2024-11-10T11:00:42,545 INFO [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C36361%2C1731236441389.meta:.meta(num 1731236442213) 2024-11-10T11:00:42,545 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,545 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,545 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,546 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,546 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741833_1009 (size=93) 2024-11-10T11:00:42,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741833_1009 (size=93) 2024-11-10T11:00:42,550 DEBUG [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/oldWALs 2024-11-10T11:00:42,550 INFO [RS:0;eccb6f743e07:36361 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C36361%2C1731236441389:(num 1731236441800) 2024-11-10T11:00:42,550 DEBUG [RS:0;eccb6f743e07:36361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:42,550 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:00:42,550 INFO [RS:0;eccb6f743e07:36361 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:00:42,551 INFO [RS:0;eccb6f743e07:36361 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T11:00:42,551 INFO [RS:0;eccb6f743e07:36361 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:00:42,551 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:00:42,551 INFO [RS:0;eccb6f743e07:36361 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36361 2024-11-10T11:00:42,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,36361,1731236441389 2024-11-10T11:00:42,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:00:42,553 INFO [RS:0;eccb6f743e07:36361 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:00:42,554 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,36361,1731236441389] 2024-11-10T11:00:42,556 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,36361,1731236441389 already deleted, retry=false 2024-11-10T11:00:42,556 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,36361,1731236441389 expired; onlineServers=0 2024-11-10T11:00:42,556 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,40551,1731236441334' ***** 2024-11-10T11:00:42,556 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:00:42,556 INFO [M:0;eccb6f743e07:40551 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:00:42,556 INFO [M:0;eccb6f743e07:40551 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:00:42,556 DEBUG [M:0;eccb6f743e07:40551 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:00:42,556 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:00:42,556 DEBUG [M:0;eccb6f743e07:40551 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:00:42,556 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236441589 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236441589,5,FailOnTimeoutGroup] 2024-11-10T11:00:42,556 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236441590 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236441590,5,FailOnTimeoutGroup] 2024-11-10T11:00:42,557 INFO [M:0;eccb6f743e07:40551 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:00:42,557 INFO [M:0;eccb6f743e07:40551 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:00:42,557 DEBUG [M:0;eccb6f743e07:40551 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:00:42,557 INFO [M:0;eccb6f743e07:40551 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:00:42,557 INFO [M:0;eccb6f743e07:40551 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:00:42,557 INFO [M:0;eccb6f743e07:40551 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:00:42,557 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:00:42,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:00:42,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:42,559 DEBUG [M:0;eccb6f743e07:40551 {}] zookeeper.ZKUtil(347): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:00:42,559 WARN [M:0;eccb6f743e07:40551 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:00:42,560 INFO [M:0;eccb6f743e07:40551 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/.lastflushedseqids 2024-11-10T11:00:42,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741836_1012 (size=99) 2024-11-10T11:00:42,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741836_1012 (size=99) 2024-11-10T11:00:42,567 INFO [M:0;eccb6f743e07:40551 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:00:42,567 INFO [M:0;eccb6f743e07:40551 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:00:42,567 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:00:42,567 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:42,567 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:42,568 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:00:42,568 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:42,568 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-10T11:00:42,586 DEBUG [M:0;eccb6f743e07:40551 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/79cecf5b9ed040ffafa599643161efaa is 82, key is hbase:meta,,1/info:regioninfo/1731236442242/Put/seqid=0 2024-11-10T11:00:42,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741837_1013 (size=5672) 2024-11-10T11:00:42,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741837_1013 (size=5672) 2024-11-10T11:00:42,592 INFO [M:0;eccb6f743e07:40551 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/79cecf5b9ed040ffafa599643161efaa 2024-11-10T11:00:42,616 DEBUG [M:0;eccb6f743e07:40551 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95e0e37513814a189ee1696f56ed9b33 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731236442269/Put/seqid=0 2024-11-10T11:00:42,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741838_1014 (size=5275) 2024-11-10T11:00:42,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741838_1014 (size=5275) 2024-11-10T11:00:42,623 INFO [M:0;eccb6f743e07:40551 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95e0e37513814a189ee1696f56ed9b33 2024-11-10T11:00:42,645 DEBUG [M:0;eccb6f743e07:40551 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58306f9bbd994289b2440c69fa35990d is 69, key is eccb6f743e07,36361,1731236441389/rs:state/1731236441640/Put/seqid=0 2024-11-10T11:00:42,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741839_1015 (size=5156) 2024-11-10T11:00:42,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741839_1015 (size=5156) 2024-11-10T11:00:42,652 INFO [M:0;eccb6f743e07:40551 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58306f9bbd994289b2440c69fa35990d 2024-11-10T11:00:42,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:42,655 INFO [RS:0;eccb6f743e07:36361 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:00:42,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36361-0x101018997730001, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:42,655 INFO [RS:0;eccb6f743e07:36361 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,36361,1731236441389; zookeeper connection closed. 2024-11-10T11:00:42,655 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@32b37b48 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@32b37b48 2024-11-10T11:00:42,655 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T11:00:42,674 DEBUG [M:0;eccb6f743e07:40551 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c90f53f43fdf414ba8f061efc870892d is 52, key is load_balancer_on/state:d/1731236442330/Put/seqid=0 2024-11-10T11:00:42,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741840_1016 (size=5056) 2024-11-10T11:00:42,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741840_1016 (size=5056) 2024-11-10T11:00:42,680 INFO [M:0;eccb6f743e07:40551 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c90f53f43fdf414ba8f061efc870892d 2024-11-10T11:00:42,688 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/79cecf5b9ed040ffafa599643161efaa as hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/79cecf5b9ed040ffafa599643161efaa 2024-11-10T11:00:42,694 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/79cecf5b9ed040ffafa599643161efaa, entries=8, sequenceid=29, filesize=5.5 K 2024-11-10T11:00:42,695 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/95e0e37513814a189ee1696f56ed9b33 as hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95e0e37513814a189ee1696f56ed9b33 2024-11-10T11:00:42,701 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/95e0e37513814a189ee1696f56ed9b33, entries=3, sequenceid=29, filesize=5.2 K 2024-11-10T11:00:42,703 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/58306f9bbd994289b2440c69fa35990d as hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58306f9bbd994289b2440c69fa35990d 2024-11-10T11:00:42,709 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/58306f9bbd994289b2440c69fa35990d, entries=1, sequenceid=29, filesize=5.0 K 2024-11-10T11:00:42,710 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c90f53f43fdf414ba8f061efc870892d as hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c90f53f43fdf414ba8f061efc870892d 2024-11-10T11:00:42,716 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38271/user/jenkins/test-data/3f8b678a-8ba7-61ba-9c6f-2c6f49163b7e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c90f53f43fdf414ba8f061efc870892d, entries=1, sequenceid=29, filesize=4.9 K 2024-11-10T11:00:42,718 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=29, compaction requested=false 2024-11-10T11:00:42,719 INFO [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:42,720 DEBUG [M:0;eccb6f743e07:40551 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236442567Disabling compacts and flushes for region at 1731236442567Disabling writes for close at 1731236442568 (+1 ms)Obtaining lock to block concurrent updates at 1731236442568Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236442568Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731236442568Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236442569 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236442569Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236442585 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236442585Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236442599 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236442615 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236442615Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236442629 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236442645 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236442645Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236442658 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236442674 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236442674Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58995ea6: reopening flushed file at 1731236442687 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@dbc4d7b: reopening flushed file at 1731236442694 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3545acca: reopening flushed file at 1731236442702 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b8be202: reopening flushed file at 1731236442709 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 149ms, sequenceid=29, compaction requested=false at 1731236442718 (+9 ms)Writing region close event to WAL at 1731236442719 (+1 ms)Closed at 1731236442719 2024-11-10T11:00:42,720 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,720 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,720 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,720 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,721 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:42,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45447 is added to blk_1073741830_1006 (size=10311) 2024-11-10T11:00:42,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45191 is added to blk_1073741830_1006 (size=10311) 2024-11-10T11:00:42,724 INFO [M:0;eccb6f743e07:40551 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:00:42,724 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:00:42,724 INFO [M:0;eccb6f743e07:40551 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40551 2024-11-10T11:00:42,725 INFO [M:0;eccb6f743e07:40551 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:00:42,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:42,827 INFO [M:0;eccb6f743e07:40551 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:00:42,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40551-0x101018997730000, quorum=127.0.0.1:61774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:00:42,829 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@18492d7d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:42,830 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30ebe7e3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:42,830 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:42,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@274298f3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:42,830 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@194f043a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:42,832 WARN [BP-2087770100-172.17.0.2-1731236440597 heartbeating to localhost/127.0.0.1:38271 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:42,832 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:42,832 WARN [BP-2087770100-172.17.0.2-1731236440597 heartbeating to localhost/127.0.0.1:38271 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2087770100-172.17.0.2-1731236440597 (Datanode Uuid 516eb0bb-9696-4ae7-976c-54ac0ff2d780) service to localhost/127.0.0.1:38271 2024-11-10T11:00:42,832 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:42,832 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data3/current/BP-2087770100-172.17.0.2-1731236440597 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:42,833 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data4/current/BP-2087770100-172.17.0.2-1731236440597 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:42,833 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:42,835 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a15ed6a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:42,835 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@b5aedfa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:42,835 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:42,835 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@469dec96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:42,836 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@69a0f3c6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:42,837 WARN [BP-2087770100-172.17.0.2-1731236440597 heartbeating to localhost/127.0.0.1:38271 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:42,837 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:42,837 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:42,837 WARN [BP-2087770100-172.17.0.2-1731236440597 heartbeating to localhost/127.0.0.1:38271 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2087770100-172.17.0.2-1731236440597 (Datanode Uuid 1800d232-93fb-4b26-b5a0-7cfc586a9719) service to localhost/127.0.0.1:38271 2024-11-10T11:00:42,838 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data1/current/BP-2087770100-172.17.0.2-1731236440597 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:42,838 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/cluster_061b082b-ca03-344e-f21f-be77b60c3f48/data/data2/current/BP-2087770100-172.17.0.2-1731236440597 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:42,838 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:42,848 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1edca743{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:00:42,849 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c443180{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:42,849 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:42,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@59505eb5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:42,849 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46d26a79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:42,856 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:00:42,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:00:42,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T11:00:42,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.log.dir so I do NOT create it in target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177 2024-11-10T11:00:42,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/5cacc94b-1ad1-30f7-2b43-12c133a0a376/hadoop.tmp.dir so I do NOT create it in target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450, deleteOnExit=true 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/test.cache.data in system properties and HBase conf 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir in system properties and HBase conf 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T11:00:42,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T11:00:42,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T11:00:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/nfs.dump.dir in system properties and HBase conf 2024-11-10T11:00:42,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir in system properties and HBase conf 2024-11-10T11:00:42,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:00:42,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T11:00:42,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T11:00:42,888 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:00:42,966 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:42,972 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:42,973 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:42,973 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:42,974 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:00:42,974 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:42,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16369da1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:42,975 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5e7025d4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:43,092 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cf515b1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-45457-hadoop-hdfs-3_4_1-tests_jar-_-any-5086851844817142843/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:00:43,092 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@c6abea1{HTTP/1.1, (http/1.1)}{localhost:45457} 2024-11-10T11:00:43,093 INFO [Time-limited test {}] server.Server(415): Started @103663ms 2024-11-10T11:00:43,107 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:00:43,184 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:43,187 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:43,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:43,188 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:43,188 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:00:43,191 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35a03ba9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:43,191 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5bb4f47b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:43,309 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@f1f9cf1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-35219-hadoop-hdfs-3_4_1-tests_jar-_-any-9490192578240421227/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:43,309 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2df55a{HTTP/1.1, (http/1.1)}{localhost:35219} 2024-11-10T11:00:43,309 INFO [Time-limited test {}] server.Server(415): Started @103880ms 2024-11-10T11:00:43,311 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:43,343 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:43,347 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:43,348 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:43,348 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:43,348 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:00:43,348 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d3d4ef0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:43,349 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39a69c39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:43,428 WARN [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data2/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:43,428 WARN [Thread-656 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data1/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:43,452 WARN [Thread-635 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:43,455 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2d2d3220c437adf with lease ID 0xcbdc67de8ca92ed9: Processing first storage report for DS-ff46be74-bfca-407b-904b-d377d604a52d from datanode DatanodeRegistration(127.0.0.1:34089, datanodeUuid=c9ab919d-8f3c-4fa7-80d1-58ffaa13d808, infoPort=44989, infoSecurePort=0, ipcPort=34949, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:43,455 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2d2d3220c437adf with lease ID 0xcbdc67de8ca92ed9: from storage DS-ff46be74-bfca-407b-904b-d377d604a52d node DatanodeRegistration(127.0.0.1:34089, datanodeUuid=c9ab919d-8f3c-4fa7-80d1-58ffaa13d808, infoPort=44989, infoSecurePort=0, ipcPort=34949, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:43,456 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2d2d3220c437adf with lease ID 0xcbdc67de8ca92ed9: Processing first storage report for DS-07cda21c-ab2b-4f0c-a2d8-62278bfe4fe0 from datanode DatanodeRegistration(127.0.0.1:34089, datanodeUuid=c9ab919d-8f3c-4fa7-80d1-58ffaa13d808, infoPort=44989, infoSecurePort=0, ipcPort=34949, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:43,456 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2d2d3220c437adf with lease ID 0xcbdc67de8ca92ed9: from storage DS-07cda21c-ab2b-4f0c-a2d8-62278bfe4fe0 node DatanodeRegistration(127.0.0.1:34089, datanodeUuid=c9ab919d-8f3c-4fa7-80d1-58ffaa13d808, infoPort=44989, infoSecurePort=0, ipcPort=34949, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T11:00:43,467 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@52be898{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-42551-hadoop-hdfs-3_4_1-tests_jar-_-any-291412848032436349/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:43,468 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@520d17ab{HTTP/1.1, (http/1.1)}{localhost:42551} 2024-11-10T11:00:43,468 INFO [Time-limited test {}] server.Server(415): Started @104039ms 2024-11-10T11:00:43,470 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:43,577 WARN [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data4/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:43,577 WARN [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data3/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:43,596 WARN [Thread-671 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:43,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc3a8c03fda85f83d with lease ID 0xcbdc67de8ca92eda: Processing first storage report for DS-4e37e353-e3be-40ea-b636-273dd600cc83 from datanode DatanodeRegistration(127.0.0.1:35953, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=43259, infoSecurePort=0, ipcPort=45393, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:43,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc3a8c03fda85f83d with lease ID 0xcbdc67de8ca92eda: from storage DS-4e37e353-e3be-40ea-b636-273dd600cc83 node DatanodeRegistration(127.0.0.1:35953, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=43259, infoSecurePort=0, ipcPort=45393, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:43,599 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc3a8c03fda85f83d with lease ID 0xcbdc67de8ca92eda: Processing first storage report for DS-7fb9839c-7d92-4802-8c78-fc940dce02e4 from datanode DatanodeRegistration(127.0.0.1:35953, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=43259, infoSecurePort=0, ipcPort=45393, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:43,599 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc3a8c03fda85f83d with lease ID 0xcbdc67de8ca92eda: from storage DS-7fb9839c-7d92-4802-8c78-fc940dce02e4 node DatanodeRegistration(127.0.0.1:35953, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=43259, infoSecurePort=0, ipcPort=45393, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:43,603 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177 2024-11-10T11:00:43,606 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/zookeeper_0, clientPort=54989, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T11:00:43,607 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54989 2024-11-10T11:00:43,608 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,609 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:00:43,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:00:43,621 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40 with version=8 2024-11-10T11:00:43,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T11:00:43,623 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:00:43,623 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:43,624 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:43,624 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:00:43,624 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:43,624 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:00:43,624 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T11:00:43,624 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:00:43,625 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35673 2024-11-10T11:00:43,626 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35673 connecting to ZooKeeper ensemble=127.0.0.1:54989 2024-11-10T11:00:43,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:356730x0, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:00:43,633 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35673-0x1010189a0650000 connected 2024-11-10T11:00:43,649 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,651 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,653 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:43,653 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40, hbase.cluster.distributed=false 2024-11-10T11:00:43,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:00:43,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35673 2024-11-10T11:00:43,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35673 2024-11-10T11:00:43,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35673 2024-11-10T11:00:43,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35673 2024-11-10T11:00:43,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35673 2024-11-10T11:00:43,661 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:00:43,672 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:00:43,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:43,672 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:43,673 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:00:43,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:43,673 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:00:43,673 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:00:43,673 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:00:43,673 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46705 2024-11-10T11:00:43,675 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46705 connecting to ZooKeeper ensemble=127.0.0.1:54989 2024-11-10T11:00:43,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,677 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:467050x0, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:00:43,684 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:467050x0, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:00:43,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46705-0x1010189a0650001 connected 2024-11-10T11:00:43,684 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:00:43,685 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:00:43,685 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T11:00:43,687 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:00:43,687 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46705 2024-11-10T11:00:43,688 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46705 2024-11-10T11:00:43,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46705 2024-11-10T11:00:43,690 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46705 2024-11-10T11:00:43,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46705 2024-11-10T11:00:43,713 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:35673 2024-11-10T11:00:43,713 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:43,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:43,715 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T11:00:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,717 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,718 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:00:43,718 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,35673,1731236443623 from backup master directory 2024-11-10T11:00:43,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:43,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:00:43,720 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:00:43,720 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,725 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/hbase.id] with ID: df32533c-d4c8-4312-8dc3-9a0b8ddb3b5f 2024-11-10T11:00:43,725 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/.tmp/hbase.id 2024-11-10T11:00:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:00:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:00:43,732 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/.tmp/hbase.id]:[hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/hbase.id] 2024-11-10T11:00:43,746 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:43,746 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T11:00:43,748 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-10T11:00:43,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:00:43,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:00:43,760 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:00:43,761 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T11:00:43,761 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:43,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:00:43,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:00:43,771 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store 2024-11-10T11:00:43,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:00:43,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:00:43,778 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:43,778 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:00:43,778 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:43,778 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:43,778 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:00:43,778 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:43,778 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:00:43,778 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236443778Disabling compacts and flushes for region at 1731236443778Disabling writes for close at 1731236443778Writing region close event to WAL at 1731236443778Closed at 1731236443778 2024-11-10T11:00:43,779 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/.initializing 2024-11-10T11:00:43,779 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,782 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C35673%2C1731236443623, suffix=, logDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623, archiveDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/oldWALs, maxLogs=10 2024-11-10T11:00:43,782 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C35673%2C1731236443623.1731236443782 2024-11-10T11:00:43,787 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 2024-11-10T11:00:43,788 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44989:44989),(127.0.0.1/127.0.0.1:43259:43259)] 2024-11-10T11:00:43,794 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:00:43,794 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:43,795 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,795 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T11:00:43,798 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:43,798 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,799 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T11:00:43,799 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:43,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T11:00:43,801 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:43,802 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T11:00:43,803 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,803 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:43,803 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,804 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,804 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,806 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,806 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,806 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T11:00:43,808 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:00:43,810 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:00:43,810 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752935, jitterRate=-0.042593732476234436}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T11:00:43,811 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236443795Initializing all the Stores at 1731236443796 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236443796Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236443796Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236443796Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236443796Cleaning up temporary data from old regions at 1731236443806 (+10 ms)Region opened successfully at 1731236443811 (+5 ms) 2024-11-10T11:00:43,812 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T11:00:43,815 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2070b590, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:00:43,816 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T11:00:43,816 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T11:00:43,816 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T11:00:43,816 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T11:00:43,817 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T11:00:43,817 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T11:00:43,817 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T11:00:43,820 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T11:00:43,820 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T11:00:43,822 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T11:00:43,822 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T11:00:43,823 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T11:00:43,824 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T11:00:43,824 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T11:00:43,825 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T11:00:43,826 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T11:00:43,827 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T11:00:43,833 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T11:00:43,835 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T11:00:43,845 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T11:00:43,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:43,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:00:43,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,854 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,855 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,35673,1731236443623, sessionid=0x1010189a0650000, setting cluster-up flag (Was=false) 2024-11-10T11:00:43,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,864 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T11:00:43,865 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:43,875 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T11:00:43,876 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,35673,1731236443623 2024-11-10T11:00:43,877 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T11:00:43,879 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:43,880 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T11:00:43,880 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T11:00:43,880 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,35673,1731236443623 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T11:00:43,881 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:00:43,882 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236473882 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T11:00:43,883 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T11:00:43,884 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T11:00:43,884 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T11:00:43,884 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:43,884 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T11:00:43,884 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236443884,5,FailOnTimeoutGroup] 2024-11-10T11:00:43,884 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236443884,5,FailOnTimeoutGroup] 2024-11-10T11:00:43,884 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,885 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T11:00:43,885 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,885 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,885 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,886 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T11:00:43,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:00:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:00:43,895 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T11:00:43,895 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40 2024-11-10T11:00:43,896 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(746): ClusterId : df32533c-d4c8-4312-8dc3-9a0b8ddb3b5f 2024-11-10T11:00:43,896 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:00:43,898 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:00:43,898 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:00:43,900 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:00:43,901 DEBUG [RS:0;eccb6f743e07:46705 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e5cb49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:00:43,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:00:43,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:00:43,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:43,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:00:43,906 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:00:43,906 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:43,907 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:00:43,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:00:43,909 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:43,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:00:43,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:00:43,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:43,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:00:43,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:00:43,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:43,914 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:43,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:00:43,915 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:46705 2024-11-10T11:00:43,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740 2024-11-10T11:00:43,915 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:00:43,915 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:00:43,915 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:00:43,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740 2024-11-10T11:00:43,916 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,35673,1731236443623 with port=46705, startcode=1731236443672 2024-11-10T11:00:43,916 DEBUG [RS:0;eccb6f743e07:46705 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:00:43,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:00:43,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:00:43,917 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:00:43,919 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40633, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:00:43,919 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:00:43,919 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35673 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,46705,1731236443672 2024-11-10T11:00:43,919 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35673 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,46705,1731236443672 2024-11-10T11:00:43,921 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40 2024-11-10T11:00:43,921 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39141 2024-11-10T11:00:43,921 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:00:43,922 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:00:43,922 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=780922, jitterRate=-0.0070075541734695435}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:00:43,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236443904Initializing all the Stores at 1731236443904Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236443905 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236443905Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236443905Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236443905Cleaning up temporary data from old regions at 1731236443917 (+12 ms)Region opened successfully at 1731236443923 (+6 ms) 2024-11-10T11:00:43,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:00:43,923 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:00:43,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:00:43,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:00:43,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:00:43,923 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:00:43,924 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:00:43,924 DEBUG [RS:0;eccb6f743e07:46705 {}] zookeeper.ZKUtil(111): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,46705,1731236443672 2024-11-10T11:00:43,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236443923Disabling compacts and flushes for region at 1731236443923Disabling writes for close at 1731236443923Writing region close event to WAL at 1731236443924 (+1 ms)Closed at 1731236443924 2024-11-10T11:00:43,924 WARN [RS:0;eccb6f743e07:46705 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:00:43,924 INFO [RS:0;eccb6f743e07:46705 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:43,924 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672 2024-11-10T11:00:43,924 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,46705,1731236443672] 2024-11-10T11:00:43,925 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:43,925 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T11:00:43,926 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T11:00:43,927 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:00:43,928 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:00:43,928 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T11:00:43,930 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:00:43,931 INFO [RS:0;eccb6f743e07:46705 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:00:43,931 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,931 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:00:43,932 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:00:43,932 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:00:43,932 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:00:43,933 DEBUG [RS:0;eccb6f743e07:46705 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:00:43,934 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,934 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,934 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,934 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,934 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,934 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,46705,1731236443672-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:00:43,950 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:00:43,950 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,46705,1731236443672-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,950 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,950 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.Replication(171): eccb6f743e07,46705,1731236443672 started 2024-11-10T11:00:43,965 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:43,965 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,46705,1731236443672, RpcServer on eccb6f743e07/172.17.0.2:46705, sessionid=0x1010189a0650001 2024-11-10T11:00:43,966 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:00:43,966 DEBUG [RS:0;eccb6f743e07:46705 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,46705,1731236443672 2024-11-10T11:00:43,966 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,46705,1731236443672' 2024-11-10T11:00:43,966 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:00:43,966 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:00:43,967 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:00:43,967 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:00:43,967 DEBUG [RS:0;eccb6f743e07:46705 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,46705,1731236443672 2024-11-10T11:00:43,967 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,46705,1731236443672' 2024-11-10T11:00:43,967 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:00:43,967 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:00:43,968 DEBUG [RS:0;eccb6f743e07:46705 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:00:43,968 INFO [RS:0;eccb6f743e07:46705 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:00:43,968 INFO [RS:0;eccb6f743e07:46705 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:00:44,070 INFO [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C46705%2C1731236443672, suffix=, logDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672, archiveDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs, maxLogs=32 2024-11-10T11:00:44,072 INFO [RS:0;eccb6f743e07:46705 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.1731236444071 2024-11-10T11:00:44,105 WARN [eccb6f743e07:35673 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T11:00:44,107 INFO [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 2024-11-10T11:00:44,112 DEBUG [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44989:44989),(127.0.0.1/127.0.0.1:43259:43259)] 2024-11-10T11:00:44,355 DEBUG [eccb6f743e07:35673 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T11:00:44,356 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,46705,1731236443672 2024-11-10T11:00:44,357 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,46705,1731236443672, state=OPENING 2024-11-10T11:00:44,361 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T11:00:44,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:44,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:00:44,363 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:00:44,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:44,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,46705,1731236443672}] 2024-11-10T11:00:44,363 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:44,517 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T11:00:44,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55599, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T11:00:44,523 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T11:00:44,523 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:44,525 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C46705%2C1731236443672.meta, suffix=.meta, logDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672, archiveDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs, maxLogs=32 2024-11-10T11:00:44,525 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta 2024-11-10T11:00:44,530 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta 2024-11-10T11:00:44,531 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43259:43259),(127.0.0.1/127.0.0.1:44989:44989)] 2024-11-10T11:00:44,532 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:00:44,532 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T11:00:44,533 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T11:00:44,533 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T11:00:44,533 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T11:00:44,533 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:44,533 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T11:00:44,533 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T11:00:44,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:00:44,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:00:44,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:44,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:44,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:00:44,537 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:00:44,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:44,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:44,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:00:44,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:00:44,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:44,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:44,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:00:44,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:00:44,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:44,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:00:44,541 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:00:44,541 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740 2024-11-10T11:00:44,543 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740 2024-11-10T11:00:44,544 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:00:44,544 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:00:44,544 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:00:44,546 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:00:44,546 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=876735, jitterRate=0.11482745409011841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:00:44,547 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T11:00:44,547 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236444533Writing region info on filesystem at 1731236444533Initializing all the Stores at 1731236444534 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236444534Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236444535 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236444535Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236444535Cleaning up temporary data from old regions at 1731236444544 (+9 ms)Running coprocessor post-open hooks at 1731236444547 (+3 ms)Region opened successfully at 1731236444547 2024-11-10T11:00:44,548 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236444516 2024-11-10T11:00:44,551 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T11:00:44,551 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T11:00:44,552 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,46705,1731236443672 2024-11-10T11:00:44,553 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,46705,1731236443672, state=OPEN 2024-11-10T11:00:44,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:00:44,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:00:44,558 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,46705,1731236443672 2024-11-10T11:00:44,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:44,558 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:00:44,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T11:00:44,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,46705,1731236443672 in 195 msec 2024-11-10T11:00:44,564 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T11:00:44,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 637 msec 2024-11-10T11:00:44,565 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:00:44,565 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T11:00:44,567 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:00:44,567 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,46705,1731236443672, seqNum=-1] 2024-11-10T11:00:44,567 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:00:44,569 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37547, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:00:44,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 695 msec 2024-11-10T11:00:44,575 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236444575, completionTime=-1 2024-11-10T11:00:44,575 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T11:00:44,575 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236504577 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236564577 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,35673,1731236443623-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,35673,1731236443623-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,35673,1731236443623-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:35673, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,577 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,579 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.861sec 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,35673,1731236443623-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:00:44,581 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,35673,1731236443623-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T11:00:44,584 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T11:00:44,584 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T11:00:44,584 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,35673,1731236443623-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47d1c4f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:00:44,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,35673,-1 for getting cluster id 2024-11-10T11:00:44,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T11:00:44,608 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'df32533c-d4c8-4312-8dc3-9a0b8ddb3b5f' 2024-11-10T11:00:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T11:00:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "df32533c-d4c8-4312-8dc3-9a0b8ddb3b5f" 2024-11-10T11:00:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed00ef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:00:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,35673,-1] 2024-11-10T11:00:44,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T11:00:44,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:00:44,611 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T11:00:44,612 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@597807df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:00:44,613 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:00:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,46705,1731236443672, seqNum=-1] 2024-11-10T11:00:44,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:00:44,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:00:44,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,35673,1731236443623 2024-11-10T11:00:44,618 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:44,621 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T11:00:44,637 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:00:44,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:44,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:44,637 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:00:44,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:00:44,637 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:00:44,638 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:00:44,638 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:00:44,639 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41543 2024-11-10T11:00:44,640 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41543 connecting to ZooKeeper ensemble=127.0.0.1:54989 2024-11-10T11:00:44,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:44,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:00:44,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415430x0, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:00:44,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:415430x0, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-10T11:00:44,647 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-10T11:00:44,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41543-0x1010189a0650002 connected 2024-11-10T11:00:44,648 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:00:44,650 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:00:44,651 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:00:44,652 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:00:44,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41543 2024-11-10T11:00:44,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41543 2024-11-10T11:00:44,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41543 2024-11-10T11:00:44,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41543 2024-11-10T11:00:44,658 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41543 2024-11-10T11:00:44,659 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(746): ClusterId : df32533c-d4c8-4312-8dc3-9a0b8ddb3b5f 2024-11-10T11:00:44,659 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:00:44,661 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:00:44,661 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:00:44,663 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:00:44,663 DEBUG [RS:1;eccb6f743e07:41543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62b4362e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:00:44,675 DEBUG [RS:1;eccb6f743e07:41543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;eccb6f743e07:41543 2024-11-10T11:00:44,675 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:00:44,675 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:00:44,675 DEBUG [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:00:44,676 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,35673,1731236443623 with port=41543, startcode=1731236444637 2024-11-10T11:00:44,676 DEBUG [RS:1;eccb6f743e07:41543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:00:44,678 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58235, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:00:44,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35673 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,41543,1731236444637 2024-11-10T11:00:44,678 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35673 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,41543,1731236444637 2024-11-10T11:00:44,680 DEBUG [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40 2024-11-10T11:00:44,680 DEBUG [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39141 2024-11-10T11:00:44,680 DEBUG [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:00:44,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:00:44,682 DEBUG [RS:1;eccb6f743e07:41543 {}] zookeeper.ZKUtil(111): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,41543,1731236444637 2024-11-10T11:00:44,682 WARN [RS:1;eccb6f743e07:41543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:00:44,682 INFO [RS:1;eccb6f743e07:41543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:00:44,682 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,41543,1731236444637] 2024-11-10T11:00:44,682 DEBUG [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637 2024-11-10T11:00:44,686 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:00:44,688 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:00:44,688 INFO [RS:1;eccb6f743e07:41543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:00:44,688 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,688 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:00:44,689 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:00:44,689 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,689 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,689 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,689 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:00:44,690 DEBUG [RS:1;eccb6f743e07:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:00:44,690 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,690 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,690 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,690 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,690 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,690 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41543,1731236444637-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:00:44,705 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:00:44,705 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,41543,1731236444637-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,705 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,705 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.Replication(171): eccb6f743e07,41543,1731236444637 started 2024-11-10T11:00:44,719 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:00:44,719 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,41543,1731236444637, RpcServer on eccb6f743e07/172.17.0.2:41543, sessionid=0x1010189a0650002 2024-11-10T11:00:44,719 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:00:44,719 DEBUG [RS:1;eccb6f743e07:41543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,41543,1731236444637 2024-11-10T11:00:44,719 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;eccb6f743e07:41543,5,FailOnTimeoutGroup] 2024-11-10T11:00:44,719 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,41543,1731236444637' 2024-11-10T11:00:44,719 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:00:44,720 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-11-10T11:00:44,720 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:00:44,720 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T11:00:44,720 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:00:44,720 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:00:44,720 DEBUG [RS:1;eccb6f743e07:41543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,41543,1731236444637 2024-11-10T11:00:44,720 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,41543,1731236444637' 2024-11-10T11:00:44,720 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:00:44,721 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:00:44,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is eccb6f743e07,35673,1731236443623 2024-11-10T11:00:44,721 DEBUG [RS:1;eccb6f743e07:41543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:00:44,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4606f62b 2024-11-10T11:00:44,721 INFO [RS:1;eccb6f743e07:41543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:00:44,721 INFO [RS:1;eccb6f743e07:41543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:00:44,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T11:00:44,723 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50440, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T11:00:44,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T11:00:44,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T11:00:44,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:00:44,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T11:00:44,728 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T11:00:44,728 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:44,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-10T11:00:44,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:00:44,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T11:00:44,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741835_1011 (size=393) 2024-11-10T11:00:44,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741835_1011 (size=393) 2024-11-10T11:00:44,738 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2cfec1cabbb1d72055901cb482dadc5a, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40 2024-11-10T11:00:44,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35953 is added to blk_1073741836_1012 (size=76) 2024-11-10T11:00:44,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34089 is added to blk_1073741836_1012 (size=76) 2024-11-10T11:00:44,745 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:44,745 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 2cfec1cabbb1d72055901cb482dadc5a, disabling compactions & flushes 2024-11-10T11:00:44,745 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:44,745 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:44,746 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. after waiting 0 ms 2024-11-10T11:00:44,746 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:44,746 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:44,746 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2cfec1cabbb1d72055901cb482dadc5a: Waiting for close lock at 1731236444745Disabling compacts and flushes for region at 1731236444745Disabling writes for close at 1731236444746 (+1 ms)Writing region close event to WAL at 1731236444746Closed at 1731236444746 2024-11-10T11:00:44,747 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T11:00:44,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1731236444747"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236444747"}]},"ts":"1731236444747"} 2024-11-10T11:00:44,750 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T11:00:44,751 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T11:00:44,752 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236444751"}]},"ts":"1731236444751"} 2024-11-10T11:00:44,754 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-10T11:00:44,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=2cfec1cabbb1d72055901cb482dadc5a, ASSIGN}] 2024-11-10T11:00:44,755 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=2cfec1cabbb1d72055901cb482dadc5a, ASSIGN 2024-11-10T11:00:44,757 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=2cfec1cabbb1d72055901cb482dadc5a, ASSIGN; state=OFFLINE, location=eccb6f743e07,46705,1731236443672; forceNewPlan=false, retain=false 2024-11-10T11:00:44,824 INFO [RS:1;eccb6f743e07:41543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C41543%2C1731236444637, suffix=, logDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637, archiveDir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs, maxLogs=32 2024-11-10T11:00:44,825 INFO [RS:1;eccb6f743e07:41543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C41543%2C1731236444637.1731236444825 2024-11-10T11:00:44,832 INFO [RS:1;eccb6f743e07:41543 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 2024-11-10T11:00:44,832 DEBUG [RS:1;eccb6f743e07:41543 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43259:43259),(127.0.0.1/127.0.0.1:44989:44989)] 2024-11-10T11:00:44,908 INFO [eccb6f743e07:35673 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T11:00:44,908 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2cfec1cabbb1d72055901cb482dadc5a, regionState=OPENING, regionLocation=eccb6f743e07,46705,1731236443672 2024-11-10T11:00:44,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=2cfec1cabbb1d72055901cb482dadc5a, ASSIGN because future has completed 2024-11-10T11:00:44,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfec1cabbb1d72055901cb482dadc5a, server=eccb6f743e07,46705,1731236443672}] 2024-11-10T11:00:44,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-10T11:00:45,014 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:45,016 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:45,069 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:45,069 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 2cfec1cabbb1d72055901cb482dadc5a, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:00:45,069 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,070 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:00:45,070 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,070 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,071 INFO [StoreOpener-2cfec1cabbb1d72055901cb482dadc5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,072 INFO [StoreOpener-2cfec1cabbb1d72055901cb482dadc5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2cfec1cabbb1d72055901cb482dadc5a columnFamilyName info 2024-11-10T11:00:45,073 DEBUG [StoreOpener-2cfec1cabbb1d72055901cb482dadc5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:00:45,073 INFO [StoreOpener-2cfec1cabbb1d72055901cb482dadc5a-1 {}] regionserver.HStore(327): Store=2cfec1cabbb1d72055901cb482dadc5a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:00:45,073 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,074 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,074 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,075 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,075 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,076 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,078 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:00:45,079 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 2cfec1cabbb1d72055901cb482dadc5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=843143, jitterRate=0.07211199402809143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T11:00:45,079 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:00:45,079 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 2cfec1cabbb1d72055901cb482dadc5a: Running coprocessor pre-open hook at 1731236445070Writing region info on filesystem at 1731236445070Initializing all the Stores at 1731236445071 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236445071Cleaning up temporary data from old regions at 1731236445075 (+4 ms)Running coprocessor post-open hooks at 1731236445079 (+4 ms)Region opened successfully at 1731236445079 2024-11-10T11:00:45,080 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a., pid=6, masterSystemTime=1731236445065 2024-11-10T11:00:45,083 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:45,083 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:45,084 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=2cfec1cabbb1d72055901cb482dadc5a, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,46705,1731236443672 2024-11-10T11:00:45,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2cfec1cabbb1d72055901cb482dadc5a, server=eccb6f743e07,46705,1731236443672 because future has completed 2024-11-10T11:00:45,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T11:00:45,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 2cfec1cabbb1d72055901cb482dadc5a, server=eccb6f743e07,46705,1731236443672 in 176 msec 2024-11-10T11:00:45,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T11:00:45,093 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=2cfec1cabbb1d72055901cb482dadc5a, ASSIGN in 336 msec 2024-11-10T11:00:45,094 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T11:00:45,095 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236445094"}]},"ts":"1731236445094"} 2024-11-10T11:00:45,097 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-10T11:00:45,098 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T11:00:45,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 373 msec 2024-11-10T11:00:45,526 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:00:45,528 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:45,546 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:45,546 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:45,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:49,928 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T11:00:49,929 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-10T11:00:50,471 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:00:50,472 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:50,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:50,492 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:50,493 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:00:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35673 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:00:54,738 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-10T11:00:54,738 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-10T11:00:54,742 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T11:00:54,742 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:00:54,759 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:54,762 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:54,763 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:54,763 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:54,763 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:00:54,765 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b53e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:54,765 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@402062d6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:54,893 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7a2a3a4f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-43949-hadoop-hdfs-3_4_1-tests_jar-_-any-7937887374093161163/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:54,894 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7fa27241{HTTP/1.1, (http/1.1)}{localhost:43949} 2024-11-10T11:00:54,894 INFO [Time-limited test {}] server.Server(415): Started @115465ms 2024-11-10T11:00:54,895 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:54,931 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:54,934 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:54,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:54,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:54,935 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:00:54,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ca4b7c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:54,936 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c5e4864{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:54,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T11:00:54,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-10T11:00:55,008 WARN [Thread-828 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:55,008 WARN [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:55,038 WARN [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:55,041 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb549fb3854a2756a with lease ID 0xcbdc67de8ca92edb: Processing first storage report for DS-906c57e6-b836-458c-ab59-619e9a182ef7 from datanode DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:55,041 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb549fb3854a2756a with lease ID 0xcbdc67de8ca92edb: from storage DS-906c57e6-b836-458c-ab59-619e9a182ef7 node DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:55,042 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb549fb3854a2756a with lease ID 0xcbdc67de8ca92edb: Processing first storage report for DS-27878d69-675c-4c9a-a7f1-d83ca2fd6a57 from datanode DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:55,042 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb549fb3854a2756a with lease ID 0xcbdc67de8ca92edb: from storage DS-27878d69-675c-4c9a-a7f1-d83ca2fd6a57 node DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:55,054 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1d80e9a4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-37483-hadoop-hdfs-3_4_1-tests_jar-_-any-8738503363508730278/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:55,054 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2c590722{HTTP/1.1, (http/1.1)}{localhost:37483} 2024-11-10T11:00:55,055 INFO [Time-limited test {}] server.Server(415): Started @115625ms 2024-11-10T11:00:55,056 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:55,096 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:00:55,101 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:00:55,105 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:00:55,105 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:00:55,105 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:00:55,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c141b19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:00:55,106 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@25509568{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:00:55,160 WARN [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data8/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:55,160 WARN [Thread-863 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data7/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:55,196 WARN [Thread-843 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:55,199 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ae94d0069f7c579 with lease ID 0xcbdc67de8ca92edc: Processing first storage report for DS-ec87db15-bdb7-4c78-8830-7c14672078ab from datanode DatanodeRegistration(127.0.0.1:34873, datanodeUuid=92b073e6-9334-4680-ad7c-b04d29c2bc4a, infoPort=44119, infoSecurePort=0, ipcPort=42191, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:55,199 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ae94d0069f7c579 with lease ID 0xcbdc67de8ca92edc: from storage DS-ec87db15-bdb7-4c78-8830-7c14672078ab node DatanodeRegistration(127.0.0.1:34873, datanodeUuid=92b073e6-9334-4680-ad7c-b04d29c2bc4a, infoPort=44119, infoSecurePort=0, ipcPort=42191, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:55,199 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8ae94d0069f7c579 with lease ID 0xcbdc67de8ca92edc: Processing first storage report for DS-1f8161c7-6fed-4d9d-9f15-6f86a55c1ba7 from datanode DatanodeRegistration(127.0.0.1:34873, datanodeUuid=92b073e6-9334-4680-ad7c-b04d29c2bc4a, infoPort=44119, infoSecurePort=0, ipcPort=42191, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:55,199 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8ae94d0069f7c579 with lease ID 0xcbdc67de8ca92edc: from storage DS-1f8161c7-6fed-4d9d-9f15-6f86a55c1ba7 node DatanodeRegistration(127.0.0.1:34873, datanodeUuid=92b073e6-9334-4680-ad7c-b04d29c2bc4a, infoPort=44119, infoSecurePort=0, ipcPort=42191, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:55,234 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5b3e7853{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-44919-hadoop-hdfs-3_4_1-tests_jar-_-any-4165249811624379513/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:55,234 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@65429201{HTTP/1.1, (http/1.1)}{localhost:44919} 2024-11-10T11:00:55,234 INFO [Time-limited test {}] server.Server(415): Started @115805ms 2024-11-10T11:00:55,236 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:00:55,337 WARN [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data9/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:55,337 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data10/current/BP-1123061454-172.17.0.2-1731236442906/current, will proceed with Du for space computation calculation, 2024-11-10T11:00:55,358 WARN [Thread-878 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:00:55,361 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xddfe6b21e21baa1d with lease ID 0xcbdc67de8ca92edd: Processing first storage report for DS-059cda56-8690-4927-8f73-78f2256c86bc from datanode DatanodeRegistration(127.0.0.1:46585, datanodeUuid=59b84983-015e-49cf-9508-73092345d0ae, infoPort=44389, infoSecurePort=0, ipcPort=41821, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:55,361 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xddfe6b21e21baa1d with lease ID 0xcbdc67de8ca92edd: from storage DS-059cda56-8690-4927-8f73-78f2256c86bc node DatanodeRegistration(127.0.0.1:46585, datanodeUuid=59b84983-015e-49cf-9508-73092345d0ae, infoPort=44389, infoSecurePort=0, ipcPort=41821, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:55,361 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xddfe6b21e21baa1d with lease ID 0xcbdc67de8ca92edd: Processing first storage report for DS-758dd0f4-7082-4062-bf77-1a501800a245 from datanode DatanodeRegistration(127.0.0.1:46585, datanodeUuid=59b84983-015e-49cf-9508-73092345d0ae, infoPort=44389, infoSecurePort=0, ipcPort=41821, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906) 2024-11-10T11:00:55,361 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xddfe6b21e21baa1d with lease ID 0xcbdc67de8ca92edd: from storage DS-758dd0f4-7082-4062-bf77-1a501800a245 node DatanodeRegistration(127.0.0.1:46585, datanodeUuid=59b84983-015e-49cf-9508-73092345d0ae, infoPort=44389, infoSecurePort=0, ipcPort=41821, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:00:55,371 WARN [ResponseProcessor for block BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,371 WARN [ResponseProcessor for block BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,371 WARN [ResponseProcessor for block BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,371 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta block BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:00:55,371 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:00:55,372 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:00:55,371 WARN [ResponseProcessor for block BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,372 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:00:55,372 WARN [PacketResponder: BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35953] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,373 WARN [PacketResponder: BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:35953] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,373 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@52be898{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:55,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:41792 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41792 dst: /127.0.0.1:34089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,373 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:41818 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41818 dst: /127.0.0.1:34089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,374 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@520d17ab{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:55,374 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:55,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39a69c39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:55,374 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d3d4ef0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:55,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:35868 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:35953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35868 dst: /127.0.0.1:35953 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:41822 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41822 dst: /127.0.0.1:34089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,376 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:55,376 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1123061454-172.17.0.2-1731236442906 (Datanode Uuid 1fef1aa5-b235-4cbe-98ce-a6a9de5795ad) service to localhost/127.0.0.1:39141 2024-11-10T11:00:55,375 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:35856 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:35953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35856 dst: /127.0.0.1:35953 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2018523630_22 at /127.0.0.1:35914 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:35953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35914 dst: /127.0.0.1:35953 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,375 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2018523630_22 at /127.0.0.1:41848 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:34089:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41848 dst: /127.0.0.1:34089 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,374 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:35880 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:35953:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35880 dst: /127.0.0.1:35953 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:55,377 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data4/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:55,378 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data3/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:55,378 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:55,382 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:55,377 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:55,384 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,386 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,386 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta block BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,386 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@f1f9cf1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:55,387 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,387 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2df55a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:55,387 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:55,387 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5bb4f47b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:55,387 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35a03ba9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:55,388 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:55,388 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1123061454-172.17.0.2-1731236442906 (Datanode Uuid c9ab919d-8f3c-4fa7-80d1-58ffaa13d808) service to localhost/127.0.0.1:39141 2024-11-10T11:00:55,388 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:55,388 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:55,389 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data1/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:55,389 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data2/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:55,389 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:55,393 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a., hostname=eccb6f743e07,46705,1731236443672, seqNum=2] 2024-11-10T11:00:55,394 ERROR [FSHLog-0-hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40-prefix:eccb6f743e07,46705,1731236443672 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,394 WARN [FSHLog-0-hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40-prefix:eccb6f743e07,46705,1731236443672 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,395 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,395 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C46705%2C1731236443672:(num 1731236444071) roll requested 2024-11-10T11:00:55,395 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.1731236455395 2024-11-10T11:00:55,401 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:55,401 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:55,401 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:55,401 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:55,401 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:55,401 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236455395 2024-11-10T11:00:55,402 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,402 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:55,402 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44389:44389),(127.0.0.1/127.0.0.1:44119:44119)] 2024-11-10T11:00:55,402 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:00:55,403 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-10T11:00:55,403 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-10T11:00:55,403 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 2024-11-10T11:00:55,406 WARN [IPC Server handler 0 on default port 39141 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009 2024-11-10T11:00:55,410 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 after 5ms 2024-11-10T11:00:55,762 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:56,691 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:57,403 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:57,404 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236455395 2024-11-10T11:00:57,404 WARN [ResponseProcessor for block BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:57,405 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236455395 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:00:57,405 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:38564 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:46585:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38564 dst: /127.0.0.1:46585 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:57,405 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:34964 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:34873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34964 dst: /127.0.0.1:34873 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:57,407 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5b3e7853{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:00:57,407 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@65429201{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:00:57,408 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:00:57,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@25509568{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:00:57,408 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c141b19{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:00:57,410 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:00:57,410 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:00:57,410 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:00:57,410 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1123061454-172.17.0.2-1731236442906 (Datanode Uuid 59b84983-015e-49cf-9508-73092345d0ae) service to localhost/127.0.0.1:39141 2024-11-10T11:00:57,411 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data9/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:57,411 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data10/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:00:57,411 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:00:57,763 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:58,691 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:59,403 WARN [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]] 2024-11-10T11:00:59,403 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:59,404 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C46705%2C1731236443672:(num 1731236455395) roll requested 2024-11-10T11:00:59,404 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.1731236459404 2024-11-10T11:00:59,407 WARN [Thread-909 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741839_1021 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:59,407 WARN [Thread-909 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:00:59,407 WARN [Thread-909 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741839_1021 2024-11-10T11:00:59,410 WARN [Thread-909 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:00:59,410 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 after 4007ms 2024-11-10T11:00:59,415 WARN [Thread-909 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1022 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34089 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:59,415 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:34986 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741840_1022] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data8]'}, localName='127.0.0.1:34873', datanodeUuid='92b073e6-9334-4680-ad7c-b04d29c2bc4a', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741840_1022 to mirror 127.0.0.1:34089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:59,415 WARN [Thread-909 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741840_1022 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:00:59,415 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:34986 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741840_1022] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T11:00:59,415 WARN [Thread-909 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741840_1022 2024-11-10T11:00:59,415 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:34986 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741840_1022] {}] datanode.DataXceiver(331): 127.0.0.1:34873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34986 dst: /127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:00:59,416 WARN [Thread-909 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:00:59,416 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T11:00:59,420 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:59,420 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:59,420 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:59,420 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:59,420 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:00:59,421 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236455395 with entries=3, filesize=3.51 KB; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236459404 2024-11-10T11:00:59,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34873 is added to blk_1073741838_1020 (size=3600) 2024-11-10T11:00:59,425 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37329:37329),(127.0.0.1/127.0.0.1:44119:44119)] 2024-11-10T11:00:59,425 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:00:59,425 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236455395 is not closed yet, will try archiving it next time 2024-11-10T11:00:59,763 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:00:59,823 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:01:00,692 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,211 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@57e35ef0[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:34873, datanodeUuid=92b073e6-9334-4680-ad7c-b04d29c2bc4a, infoPort=44119, infoSecurePort=0, ipcPort=42191, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741838_1020 to 127.0.0.1:46585 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,420 WARN [ResponseProcessor for block BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023 java.io.IOException: Bad response ERROR for BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023 from datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,421 WARN [DataStreamer for file /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236459404 block BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:01,421 WARN [PacketResponder: BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:34873] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,421 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:39384 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39384 dst: /127.0.0.1:43619 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,422 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:35000 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1023] {}] datanode.DataXceiver(331): 127.0.0.1:34873:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35000 dst: /127.0.0.1:34873 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1d80e9a4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:01,425 WARN [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]] 2024-11-10T11:01:01,425 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2c590722{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:01,425 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:01,425 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,425 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C46705%2C1731236443672:(num 1731236459404) roll requested 2024-11-10T11:01:01,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c5e4864{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:01,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ca4b7c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:01,426 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.1731236461425 2024-11-10T11:01:01,428 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:01,428 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:01,428 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1123061454-172.17.0.2-1731236442906 (Datanode Uuid 92b073e6-9334-4680-ad7c-b04d29c2bc4a) service to localhost/127.0.0.1:39141 2024-11-10T11:01:01,428 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:01,428 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data7/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:01,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data8/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:01,430 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,430 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:01,430 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741842_1025 2024-11-10T11:01:01,431 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:01,432 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,432 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:01,432 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741843_1026 2024-11-10T11:01:01,433 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:01,435 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741844_1027 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,435 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741844_1027 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:01,435 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741844_1027 2024-11-10T11:01:01,436 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:01,438 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:01,441 WARN [Thread-921 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34089 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,441 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53840 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741845_1028] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741845_1028 to mirror 127.0.0.1:34089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,442 WARN [Thread-921 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:01,442 WARN [Thread-921 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741845_1028 2024-11-10T11:01:01,442 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53840 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741845_1028] {}] datanode.BlockReceiver(316): Block 1073741845 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T11:01:01,442 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53840 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741845_1028] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53840 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,444 WARN [Thread-921 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:01,445 WARN [IPC Server handler 2 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:01,446 WARN [IPC Server handler 2 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:01,446 WARN [IPC Server handler 2 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46705 {}] regionserver.HRegion(8855): Flush requested on 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:01,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2cfec1cabbb1d72055901cb482dadc5a 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:01:01,457 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:01,457 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:01,457 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:01,458 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:01,458 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:01,459 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236459404 with entries=4, filesize=4.65 KB; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236461425 2024-11-10T11:01:01,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741841_1024 (size=4769) 2024-11-10T11:01:01,465 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37329:37329)] 2024-11-10T11:01:01,465 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:01:01,465 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236459404 is not closed yet, will try archiving it next time 2024-11-10T11:01:01,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/5a929948b8d2406193e09260c34525e2 is 1080, key is row0002/info:/1731236457413/Put/seqid=0 2024-11-10T11:01:01,499 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,500 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:01,500 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741847_1030 2024-11-10T11:01:01,501 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:01,503 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35953 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,503 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53866 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741848_1031 to mirror 127.0.0.1:35953 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,503 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53866 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:01,504 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:01,504 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741848_1031 2024-11-10T11:01:01,504 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53866 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53866 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,504 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:01,506 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741849_1032 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,506 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741849_1032 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:01,506 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741849_1032 2024-11-10T11:01:01,508 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:01,511 WARN [Thread-926 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34089 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53874 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741850_1033] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741850_1033 to mirror 127.0.0.1:34089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,511 WARN [Thread-926 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:01,511 WARN [Thread-926 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741850_1033 2024-11-10T11:01:01,511 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53874 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741850_1033] {}] datanode.BlockReceiver(316): Block 1073741850 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:01,511 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53874 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741850_1033] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53874 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:01,512 WARN [Thread-926 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:01,513 WARN [IPC Server handler 0 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:01,513 WARN [IPC Server handler 0 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:01,513 WARN [IPC Server handler 0 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741851_1034 (size=10347) 2024-11-10T11:01:01,764 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:01,862 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:01:01,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/5a929948b8d2406193e09260c34525e2 2024-11-10T11:01:01,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/5a929948b8d2406193e09260c34525e2 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5a929948b8d2406193e09260c34525e2 2024-11-10T11:01:01,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5a929948b8d2406193e09260c34525e2, entries=5, sequenceid=11, filesize=10.1 K 2024-11-10T11:01:01,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 2cfec1cabbb1d72055901cb482dadc5a in 478ms, sequenceid=11, compaction requested=false 2024-11-10T11:01:01,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46705 {}] regionserver.HRegion(8855): Flush requested on 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:02,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2cfec1cabbb1d72055901cb482dadc5a 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-11-10T11:01:02,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/73bded3e20eb4fb5b8ef42db0c858f14 is 1080, key is row0007/info:/1731236461458/Put/seqid=0 2024-11-10T11:01:02,101 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34089 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:02,101 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53910 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741852_1035] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741852_1035 to mirror 127.0.0.1:34089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:02,101 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:02,101 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741852_1035 2024-11-10T11:01:02,101 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53910 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741852_1035] {}] datanode.BlockReceiver(316): Block 1073741852 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:02,101 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53910 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741852_1035] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53910 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:02,102 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:02,104 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35953 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:02,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53912 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741853_1036] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741853_1036 to mirror 127.0.0.1:35953 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:02,104 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:02,104 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741853_1036 2024-11-10T11:01:02,104 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53912 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741853_1036] {}] datanode.BlockReceiver(316): Block 1073741853 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:02,104 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53912 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741853_1036] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53912 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:02,104 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:02,107 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741854_1037 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:02,107 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741854_1037 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:02,106 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53922 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741854_1037] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741854_1037 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:02,107 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741854_1037 2024-11-10T11:01:02,107 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53922 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741854_1037] {}] datanode.BlockReceiver(316): Block 1073741854 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:02,107 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53922 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741854_1037] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53922 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:02,107 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:02,108 WARN [Thread-933 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:02,108 WARN [Thread-933 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:02,108 WARN [Thread-933 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741855_1038 2024-11-10T11:01:02,109 WARN [Thread-933 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:02,109 WARN [IPC Server handler 3 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:02,109 WARN [IPC Server handler 3 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:02,110 WARN [IPC Server handler 3 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:02,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741856_1039 (size=12506) 2024-11-10T11:01:02,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/73bded3e20eb4fb5b8ef42db0c858f14 2024-11-10T11:01:02,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/73bded3e20eb4fb5b8ef42db0c858f14 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14 2024-11-10T11:01:02,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14, entries=7, sequenceid=24, filesize=12.2 K 2024-11-10T11:01:02,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 2cfec1cabbb1d72055901cb482dadc5a in 435ms, sequenceid=24, compaction requested=false 2024-11-10T11:01:02,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:02,528 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-11-10T11:01:02,528 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:02,529 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14 because midkey is the same as first or last row 2024-11-10T11:01:02,692 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,465 WARN [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]] 2024-11-10T11:01:03,465 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,465 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C46705%2C1731236443672:(num 1731236461425) roll requested 2024-11-10T11:01:03,466 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.1731236463466 2024-11-10T11:01:03,469 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,469 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:03,469 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741857_1040 2024-11-10T11:01:03,469 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:03,470 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,471 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:03,471 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741858_1041 2024-11-10T11:01:03,471 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:03,472 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741859_1042 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,472 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741859_1042 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:03,472 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741859_1042 2024-11-10T11:01:03,473 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:03,474 WARN [Thread-940 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,474 WARN [Thread-940 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:03,474 WARN [Thread-940 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741860_1043 2024-11-10T11:01:03,475 WARN [Thread-940 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:03,475 WARN [IPC Server handler 1 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:03,475 WARN [IPC Server handler 1 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:03,476 WARN [IPC Server handler 1 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:03,478 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:03,478 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:03,478 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:03,478 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:03,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:03,479 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236461425 with entries=20, filesize=19.67 KB; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236463466 2024-11-10T11:01:03,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741846_1029 (size=20147) 2024-11-10T11:01:03,483 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37329:37329)] 2024-11-10T11:01:03,483 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:01:03,484 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236461425 is not closed yet, will try archiving it next time 2024-11-10T11:01:03,484 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236455395 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs/eccb6f743e07%2C46705%2C1731236443672.1731236455395 2024-11-10T11:01:03,485 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236459404 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs/eccb6f743e07%2C46705%2C1731236443672.1731236459404 2024-11-10T11:01:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46705 {}] regionserver.HRegion(8855): Flush requested on 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:03,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2cfec1cabbb1d72055901cb482dadc5a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T11:01:03,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/a7ba6f3b25fa49e8a2ea6fdb1162f287 is 1079, key is tmprow/info:/1731236463510/Put/seqid=0 2024-11-10T11:01:03,518 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,519 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:03,519 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741862_1045 2024-11-10T11:01:03,519 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:03,520 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,521 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:03,521 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741863_1046 2024-11-10T11:01:03,521 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:03,523 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53948 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741864_1047] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741864_1047 to mirror 127.0.0.1:34089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,523 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741864_1047 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34089 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,524 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53948 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741864_1047] {}] datanode.BlockReceiver(316): Block 1073741864 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:03,524 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741864_1047 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:03,524 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741864_1047 2024-11-10T11:01:03,524 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53948 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741864_1047] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53948 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,524 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:03,526 WARN [Thread-944 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:46585 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,526 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53960 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741865_1048] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741865_1048 to mirror 127.0.0.1:46585 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,526 WARN [Thread-944 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:03,526 WARN [Thread-944 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741865_1048 2024-11-10T11:01:03,527 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53960 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741865_1048] {}] datanode.BlockReceiver(316): Block 1073741865 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:03,527 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53960 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741865_1048] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53960 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,527 WARN [Thread-944 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:03,528 WARN [IPC Server handler 3 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:03,528 WARN [IPC Server handler 3 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:03,528 WARN [IPC Server handler 3 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:03,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741866_1049 (size=6027) 2024-11-10T11:01:03,764 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,881 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 is not closed yet, will try archiving it next time 2024-11-10T11:01:03,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/a7ba6f3b25fa49e8a2ea6fdb1162f287 2024-11-10T11:01:03,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/a7ba6f3b25fa49e8a2ea6fdb1162f287 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/a7ba6f3b25fa49e8a2ea6fdb1162f287 2024-11-10T11:01:03,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/a7ba6f3b25fa49e8a2ea6fdb1162f287, entries=1, sequenceid=34, filesize=5.9 K 2024-11-10T11:01:03,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 2cfec1cabbb1d72055901cb482dadc5a in 432ms, sequenceid=34, compaction requested=true 2024-11-10T11:01:03,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-11-10T11:01:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14 because midkey is the same as first or last row 2024-11-10T11:01:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2cfec1cabbb1d72055901cb482dadc5a:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:01:03,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:01:03,945 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:01:03,946 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:01:03,946 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HStore(1541): 2cfec1cabbb1d72055901cb482dadc5a/info is initiating minor compaction (all files) 2024-11-10T11:01:03,946 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2cfec1cabbb1d72055901cb482dadc5a/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:03,946 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5a929948b8d2406193e09260c34525e2, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/a7ba6f3b25fa49e8a2ea6fdb1162f287] into tmpdir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp, totalSize=28.2 K 2024-11-10T11:01:03,947 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a929948b8d2406193e09260c34525e2, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731236457413 2024-11-10T11:01:03,947 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73bded3e20eb4fb5b8ef42db0c858f14, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1731236461458 2024-11-10T11:01:03,948 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7ba6f3b25fa49e8a2ea6fdb1162f287, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731236463510 2024-11-10T11:01:03,961 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2cfec1cabbb1d72055901cb482dadc5a#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:01:03,961 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/5932423355b84fc3accab2ada0b7452e is 1080, key is row0002/info:/1731236457413/Put/seqid=0 2024-11-10T11:01:03,963 WARN [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,963 WARN [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:03,963 WARN [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741867_1050 2024-11-10T11:01:03,964 WARN [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:03,965 WARN [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,965 WARN [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:03,965 WARN [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741868_1051 2024-11-10T11:01:03,965 WARN [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:03,969 WARN [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741869_1052 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:35953 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,969 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53990 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741869_1052] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741869_1052 to mirror 127.0.0.1:35953 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,969 WARN [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741869_1052 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:03,969 WARN [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741869_1052 2024-11-10T11:01:03,969 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53990 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741869_1052] {}] datanode.BlockReceiver(316): Block 1073741869 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:03,969 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53990 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741869_1052] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53990 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,969 WARN [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:03,971 WARN [Thread-952 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34089 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:03,971 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53998 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741870_1053 to mirror 127.0.0.1:34089 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,972 WARN [Thread-952 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:03,972 WARN [Thread-952 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741870_1053 2024-11-10T11:01:03,972 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53998 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:03,972 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:53998 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53998 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:03,972 WARN [Thread-952 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:03,973 WARN [IPC Server handler 2 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:03,973 WARN [IPC Server handler 2 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:03,973 WARN [IPC Server handler 2 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:03,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741871_1054 (size=17994) 2024-11-10T11:01:04,382 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/5932423355b84fc3accab2ada0b7452e as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e 2024-11-10T11:01:04,390 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2cfec1cabbb1d72055901cb482dadc5a/info of 2cfec1cabbb1d72055901cb482dadc5a into 5932423355b84fc3accab2ada0b7452e(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:01:04,390 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:04,390 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a., storeName=2cfec1cabbb1d72055901cb482dadc5a/info, priority=13, startTime=1731236463945; duration=0sec 2024-11-10T11:01:04,390 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T11:01:04,390 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:04,390 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e because midkey is the same as first or last row 2024-11-10T11:01:04,390 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e because midkey is the same as first or last row 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e because midkey is the same as first or last row 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:01:04,391 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2cfec1cabbb1d72055901cb482dadc5a:info 2024-11-10T11:01:04,692 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:04,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46705 {}] regionserver.HRegion(8855): Flush requested on 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:04,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2cfec1cabbb1d72055901cb482dadc5a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T11:01:04,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/6d9903a287a24e0e91b6cd41e35e16bc is 1079, key is tmprow/info:/1731236464929/Put/seqid=0 2024-11-10T11:01:04,936 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:04,936 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:04,936 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741872_1055 2024-11-10T11:01:04,937 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:04,938 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:04,938 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK], DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]) is bad. 2024-11-10T11:01:04,938 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741873_1056 2024-11-10T11:01:04,938 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK] 2024-11-10T11:01:04,940 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741874_1057 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:04,940 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741874_1057 in pipeline [DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK], DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK]) is bad. 2024-11-10T11:01:04,940 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741874_1057 2024-11-10T11:01:04,940 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:46585,DS-059cda56-8690-4927-8f73-78f2256c86bc,DISK] 2024-11-10T11:01:04,941 WARN [Thread-958 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:04,941 WARN [Thread-958 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]) is bad. 2024-11-10T11:01:04,941 WARN [Thread-958 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741875_1058 2024-11-10T11:01:04,942 WARN [Thread-958 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:35953,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK] 2024-11-10T11:01:04,942 WARN [IPC Server handler 0 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-10T11:01:04,942 WARN [IPC Server handler 0 on default port 39141 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-10T11:01:04,942 WARN [IPC Server handler 0 on default port 39141 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-10T11:01:04,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741876_1059 (size=6027) 2024-11-10T11:01:05,042 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6dbd1900[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741851_1034 to 127.0.0.1:34089 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:05,042 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@637afb1f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741841_1024 to 127.0.0.1:46585 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:05,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/6d9903a287a24e0e91b6cd41e35e16bc 2024-11-10T11:01:05,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/6d9903a287a24e0e91b6cd41e35e16bc as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/6d9903a287a24e0e91b6cd41e35e16bc 2024-11-10T11:01:05,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/6d9903a287a24e0e91b6cd41e35e16bc, entries=1, sequenceid=45, filesize=5.9 K 2024-11-10T11:01:05,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 2cfec1cabbb1d72055901cb482dadc5a in 429ms, sequenceid=45, compaction requested=false 2024-11-10T11:01:05,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:05,359 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-11-10T11:01:05,359 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:05,359 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e because midkey is the same as first or last row 2024-11-10T11:01:05,484 WARN [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-11-10T11:01:05,484 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:05,544 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:05,547 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:05,547 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:05,547 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:05,547 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:01:05,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@798caa79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:05,548 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@616b5398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:05,702 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2656efd1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/java.io.tmpdir/jetty-localhost-38239-hadoop-hdfs-3_4_1-tests_jar-_-any-15460579668370721377/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:05,702 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@743041f4{HTTP/1.1, (http/1.1)}{localhost:38239} 2024-11-10T11:01:05,702 INFO [Time-limited test {}] server.Server(415): Started @126273ms 2024-11-10T11:01:05,704 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:05,764 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:05,830 WARN [Thread-980 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:05,841 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x810035c2c26ea217 with lease ID 0xcbdc67de8ca92ede: from storage DS-4e37e353-e3be-40ea-b636-273dd600cc83 node DatanodeRegistration(127.0.0.1:45719, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=46591, infoSecurePort=0, ipcPort=37265, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 6, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T11:01:05,841 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x810035c2c26ea217 with lease ID 0xcbdc67de8ca92ede: from storage DS-7fb9839c-7d92-4802-8c78-fc940dce02e4 node DatanodeRegistration(127.0.0.1:45719, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=46591, infoSecurePort=0, ipcPort=37265, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:06,042 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6dbd1900[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741846_1029 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741856_1039 (size=12506) 2024-11-10T11:01:06,693 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:07,484 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:07,765 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:08,043 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@637afb1f[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741866_1049 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:08,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741871_1054 (size=17994) 2024-11-10T11:01:08,693 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:09,042 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6dbd1900[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:43619, datanodeUuid=2c9c3a51-2087-4de6-a269-0c69b1bcc5c1, infoPort=37329, infoSecurePort=0, ipcPort=36303, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741876_1059 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:09,485 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:09,765 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:10,694 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:11,485 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:11,765 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:12,694 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,486 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,603 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T11:01:13,766 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,884 ERROR [FSHLog-0-hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData-prefix:eccb6f743e07,35673,1731236443623 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,884 WARN [FSHLog-0-hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData-prefix:eccb6f743e07,35673,1731236443623 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,884 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C35673%2C1731236443623:(num 1731236443782) roll requested 2024-11-10T11:01:13,885 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C35673%2C1731236443623.1731236473884 2024-11-10T11:01:13,891 WARN [Thread-1002 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,891 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:59722 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data4]'}, localName='127.0.0.1:45719', datanodeUuid='1fef1aa5-b235-4cbe-98ce-a6a9de5795ad', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741877_1060 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:13,891 WARN [Thread-1002 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45719,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:13,891 WARN [Thread-1002 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741877_1060 2024-11-10T11:01:13,891 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:59722 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T11:01:13,891 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:59722 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:45719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59722 dst: /127.0.0.1:45719 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:13,892 WARN [Thread-1002 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:13,899 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:13,899 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:13,899 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:13,899 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:13,899 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:13,900 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236473884 2024-11-10T11:01:13,900 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,900 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:13,900 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 2024-11-10T11:01:13,901 WARN [IPC Server handler 2 on default port 39141 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 has not been closed. Lease recovery is in progress. RecoveryId = 1062 for block blk_1073741830_1006 2024-11-10T11:01:13,901 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46591:46591),(127.0.0.1/127.0.0.1:37329:37329)] 2024-11-10T11:01:13,901 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 is not closed yet, will try archiving it next time 2024-11-10T11:01:13,901 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 after 1ms 2024-11-10T11:01:14,694 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:15,486 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:15,855 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@63875847 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1123061454-172.17.0.2-1731236442906:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:34089,null,null]) java.net.ConnectException: Call From eccb6f743e07/172.17.0.2 to localhost:34949 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T11:01:15,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741833_1019 (size=455) 2024-11-10T11:01:16,427 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236444071 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs/eccb6f743e07%2C46705%2C1731236443672.1731236444071 2024-11-10T11:01:16,428 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236461425 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs/eccb6f743e07%2C46705%2C1731236443672.1731236461425 2024-11-10T11:01:16,695 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:16,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741833_1019 (size=455) 2024-11-10T11:01:17,486 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:17,902 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/WALs/eccb6f743e07,35673,1731236443623/eccb6f743e07%2C35673%2C1731236443623.1731236443782 after 4002ms 2024-11-10T11:01:18,695 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:19,487 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:19,834 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3a8a708[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45719, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=46591, infoSecurePort=0, ipcPort=37265, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741831_1007 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:19,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741835_1011 (size=393) 2024-11-10T11:01:20,695 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:20,834 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@522e5b85[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45719, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=46591, infoSecurePort=0, ipcPort=37265, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741829_1005 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:20,834 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@3a8a708[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45719, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=46591, infoSecurePort=0, ipcPort=37265, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741827_1003 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.1731236481262 2024-11-10T11:01:21,266 WARN [Thread-1017 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741879_1063 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:43366 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741879_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data4]'}, localName='127.0.0.1:45719', datanodeUuid='1fef1aa5-b235-4cbe-98ce-a6a9de5795ad', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741879_1063 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,266 WARN [Thread-1017 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741879_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45719,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,266 WARN [Thread-1017 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741879_1063 2024-11-10T11:01:21,266 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:43366 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741879_1063] {}] datanode.BlockReceiver(316): Block 1073741879 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T11:01:21,266 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-256090032_22 at /127.0.0.1:43366 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741879_1063] {}] datanode.DataXceiver(331): 127.0.0.1:45719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43366 dst: /127.0.0.1:45719 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,267 WARN [Thread-1017 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,270 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,271 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,271 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,271 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,271 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,271 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236463466 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236481262 2024-11-10T11:01:21,272 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46591:46591),(127.0.0.1/127.0.0.1:37329:37329)] 2024-11-10T11:01:21,272 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236463466 is not closed yet, will try archiving it next time 2024-11-10T11:01:21,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741861_1044 (size=13591) 2024-11-10T11:01:21,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46705 {}] regionserver.HRegion(8855): Flush requested on 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:21,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2cfec1cabbb1d72055901cb482dadc5a 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-10T11:01:21,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/e6c654497e8749d3be0ea983b5f816e2 is 1080, key is row0013/info:/1731236481273/Put/seqid=0 2024-11-10T11:01:21,289 WARN [Thread-1024 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1065 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,289 WARN [Thread-1024 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741881_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,289 WARN [Thread-1024 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741881_1065 2024-11-10T11:01:21,290 WARN [Thread-1024 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741882_1066 (size=11421) 2024-11-10T11:01:21,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741882_1066 (size=11421) 2024-11-10T11:01:21,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/e6c654497e8749d3be0ea983b5f816e2 2024-11-10T11:01:21,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/e6c654497e8749d3be0ea983b5f816e2 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/e6c654497e8749d3be0ea983b5f816e2 2024-11-10T11:01:21,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/e6c654497e8749d3be0ea983b5f816e2, entries=6, sequenceid=55, filesize=11.2 K 2024-11-10T11:01:21,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7530, heapSize ~8.11 KB/8304, currentSize=6.30 KB/6455 for 2cfec1cabbb1d72055901cb482dadc5a in 31ms, sequenceid=55, compaction requested=true 2024-11-10T11:01:21,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:21,313 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=34.6 K, sizeToCheck=16.0 K 2024-11-10T11:01:21,314 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:21,314 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e because midkey is the same as first or last row 2024-11-10T11:01:21,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2cfec1cabbb1d72055901cb482dadc5a:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:01:21,314 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:01:21,314 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:01:21,315 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:01:21,315 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HStore(1541): 2cfec1cabbb1d72055901cb482dadc5a/info is initiating minor compaction (all files) 2024-11-10T11:01:21,315 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2cfec1cabbb1d72055901cb482dadc5a/info in TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:21,315 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/6d9903a287a24e0e91b6cd41e35e16bc, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/e6c654497e8749d3be0ea983b5f816e2] into tmpdir=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp, totalSize=34.6 K 2024-11-10T11:01:21,315 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5932423355b84fc3accab2ada0b7452e, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1731236457413 2024-11-10T11:01:21,316 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d9903a287a24e0e91b6cd41e35e16bc, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1731236464929 2024-11-10T11:01:21,316 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6c654497e8749d3be0ea983b5f816e2, keycount=6, bloomtype=ROW, size=11.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731236465334 2024-11-10T11:01:21,332 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2cfec1cabbb1d72055901cb482dadc5a#info#compaction#24 average throughput is 17.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:01:21,332 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/d64b809ff4fa4e798b2bda50e49333da is 1080, key is row0002/info:/1731236457413/Put/seqid=0 2024-11-10T11:01:21,334 WARN [Thread-1033 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1067 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,334 WARN [Thread-1033 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741883_1067 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK], DatanodeInfoWithStorage[127.0.0.1:45719,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,334 WARN [Thread-1033 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741883_1067 2024-11-10T11:01:21,335 WARN [Thread-1033 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741884_1068 (size=23502) 2024-11-10T11:01:21,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741884_1068 (size=23502) 2024-11-10T11:01:21,346 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/d64b809ff4fa4e798b2bda50e49333da as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/d64b809ff4fa4e798b2bda50e49333da 2024-11-10T11:01:21,352 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2cfec1cabbb1d72055901cb482dadc5a/info of 2cfec1cabbb1d72055901cb482dadc5a into d64b809ff4fa4e798b2bda50e49333da(size=23.0 K), total size for store is 23.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:01:21,352 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2cfec1cabbb1d72055901cb482dadc5a: 2024-11-10T11:01:21,352 INFO [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a., storeName=2cfec1cabbb1d72055901cb482dadc5a/info, priority=13, startTime=1731236481314; duration=0sec 2024-11-10T11:01:21,352 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/d64b809ff4fa4e798b2bda50e49333da because midkey is the same as first or last row 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/d64b809ff4fa4e798b2bda50e49333da because midkey is the same as first or last row 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.0 K, sizeToCheck=16.0 K 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/d64b809ff4fa4e798b2bda50e49333da because midkey is the same as first or last row 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:01:21,353 DEBUG [RS:0;eccb6f743e07:46705-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2cfec1cabbb1d72055901cb482dadc5a:info 2024-11-10T11:01:21,487 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-11-10T11:01:21,487 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:01:21,493 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:01:21,494 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:21,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:21,494 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:21,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:01:21,494 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:01:21,494 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=677162162, stopped=false 2024-11-10T11:01:21,494 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,35673,1731236443623 2024-11-10T11:01:21,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:21,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:21,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:21,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:21,496 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:01:21,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:21,496 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:01:21,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:21,497 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:21,497 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:21,497 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,46705,1731236443672' ***** 2024-11-10T11:01:21,497 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:01:21,497 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,41543,1731236444637' ***** 2024-11-10T11:01:21,497 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:01:21,497 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:01:21,497 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:01:21,497 INFO [RS:0;eccb6f743e07:46705 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:01:21,497 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:01:21,497 INFO [RS:0;eccb6f743e07:46705 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:01:21,497 INFO [RS:1;eccb6f743e07:41543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:01:21,497 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:01:21,497 INFO [RS:1;eccb6f743e07:41543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:01:21,497 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,41543,1731236444637 2024-11-10T11:01:21,497 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(3091): Received CLOSE for 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:21,497 INFO [RS:1;eccb6f743e07:41543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:01:21,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:21,498 INFO [RS:1;eccb6f743e07:41543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;eccb6f743e07:41543. 2024-11-10T11:01:21,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:21,498 DEBUG [RS:1;eccb6f743e07:41543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:21,498 DEBUG [RS:1;eccb6f743e07:41543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:21,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:21,498 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,41543,1731236444637; all regions closed. 2024-11-10T11:01:21,498 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,46705,1731236443672 2024-11-10T11:01:21,498 INFO [RS:0;eccb6f743e07:46705 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:01:21,498 INFO [RS:0;eccb6f743e07:46705 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:46705. 2024-11-10T11:01:21,498 DEBUG [RS:0;eccb6f743e07:46705 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:21,498 DEBUG [RS:0;eccb6f743e07:46705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:21,498 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:01:21,498 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:01:21,498 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:01:21,498 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,499 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:01:21,499 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,499 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,499 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T11:01:21,499 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1325): Online Regions={2cfec1cabbb1d72055901cb482dadc5a=TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T11:01:21,499 DEBUG [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 2cfec1cabbb1d72055901cb482dadc5a 2024-11-10T11:01:21,499 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2cfec1cabbb1d72055901cb482dadc5a, disabling compactions & flushes 2024-11-10T11:01:21,499 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,499 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:01:21,499 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. after waiting 0 ms 2024-11-10T11:01:21,499 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:21,500 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 2cfec1cabbb1d72055901cb482dadc5a 1/1 column families, dataSize=6.30 KB heapSize=7 KB 2024-11-10T11:01:21,500 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-10T11:01:21,500 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,500 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,500 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 2024-11-10T11:01:21,500 ERROR [FSHLog-0-hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40-prefix:eccb6f743e07,46705,1731236443672.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,500 WARN [FSHLog-0-hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40-prefix:eccb6f743e07,46705,1731236443672.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,500 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C46705%2C1731236443672.meta:.meta(num 1731236444525) roll requested 2024-11-10T11:01:21,500 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46705%2C1731236443672.meta.1731236481500.meta 2024-11-10T11:01:21,501 WARN [IPC Server handler 1 on default port 39141 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 has not been closed. Lease recovery is in progress. RecoveryId = 1069 for block blk_1073741837_1013 2024-11-10T11:01:21,501 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 after 1ms 2024-11-10T11:01:21,504 WARN [Thread-1039 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741885_1070 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,504 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46800 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741885_1070] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741885_1070 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,504 WARN [Thread-1039 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741885_1070 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,504 WARN [Thread-1039 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741885_1070 2024-11-10T11:01:21,504 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46800 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741885_1070] {}] datanode.BlockReceiver(316): Block 1073741885 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-10T11:01:21,505 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46800 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741885_1070] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46800 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,505 WARN [Thread-1039 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,505 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/382dc8effabb437fa55f969da1851d43 is 1080, key is row0018/info:/1731236481283/Put/seqid=0 2024-11-10T11:01:21,512 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,513 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,513 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,513 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,513 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,513 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236481500.meta 2024-11-10T11:01:21,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741887_1072 (size=11421) 2024-11-10T11:01:21,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741887_1072 (size=11421) 2024-11-10T11:01:21,514 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,514 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.30 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/382dc8effabb437fa55f969da1851d43 2024-11-10T11:01:21,514 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34089,DS-ff46be74-bfca-407b-904b-d377d604a52d,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,514 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta 2024-11-10T11:01:21,514 WARN [IPC Server handler 3 on default port 39141 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta has not been closed. Lease recovery is in progress. RecoveryId = 1073 for block blk_1073741834_1010 2024-11-10T11:01:21,515 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta after 1ms 2024-11-10T11:01:21,515 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46591:46591),(127.0.0.1/127.0.0.1:37329:37329)] 2024-11-10T11:01:21,515 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta is not closed yet, will try archiving it next time 2024-11-10T11:01:21,521 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/.tmp/info/382dc8effabb437fa55f969da1851d43 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/382dc8effabb437fa55f969da1851d43 2024-11-10T11:01:21,527 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/382dc8effabb437fa55f969da1851d43, entries=6, sequenceid=65, filesize=11.2 K 2024-11-10T11:01:21,528 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~6.30 KB/6455, heapSize ~6.98 KB/7152, currentSize=0 B/0 for 2cfec1cabbb1d72055901cb482dadc5a in 29ms, sequenceid=65, compaction requested=false 2024-11-10T11:01:21,529 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5a929948b8d2406193e09260c34525e2, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/a7ba6f3b25fa49e8a2ea6fdb1162f287, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/6d9903a287a24e0e91b6cd41e35e16bc, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/e6c654497e8749d3be0ea983b5f816e2] to archive 2024-11-10T11:01:21,530 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T11:01:21,532 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5a929948b8d2406193e09260c34525e2 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5a929948b8d2406193e09260c34525e2 2024-11-10T11:01:21,534 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/73bded3e20eb4fb5b8ef42db0c858f14 2024-11-10T11:01:21,534 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/info/ebd6752f5cbb464c880b4d300f3f87d5 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a./info:regioninfo/1731236445084/Put/seqid=0 2024-11-10T11:01:21,535 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/5932423355b84fc3accab2ada0b7452e 2024-11-10T11:01:21,537 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/a7ba6f3b25fa49e8a2ea6fdb1162f287 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/a7ba6f3b25fa49e8a2ea6fdb1162f287 2024-11-10T11:01:21,537 WARN [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1074 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,537 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46818 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741888_1074] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741888_1074 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,537 WARN [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741888_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,537 WARN [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741888_1074 2024-11-10T11:01:21,537 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46818 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741888_1074] {}] datanode.BlockReceiver(316): Block 1073741888 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:21,538 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46818 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741888_1074] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46818 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,538 WARN [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,538 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/6d9903a287a24e0e91b6cd41e35e16bc to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/6d9903a287a24e0e91b6cd41e35e16bc 2024-11-10T11:01:21,540 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/e6c654497e8749d3be0ea983b5f816e2 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/info/e6c654497e8749d3be0ea983b5f816e2 2024-11-10T11:01:21,540 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=eccb6f743e07:35673 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-10T11:01:21,540 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [5a929948b8d2406193e09260c34525e2=10347, 73bded3e20eb4fb5b8ef42db0c858f14=12506, 5932423355b84fc3accab2ada0b7452e=17994, a7ba6f3b25fa49e8a2ea6fdb1162f287=6027, 6d9903a287a24e0e91b6cd41e35e16bc=6027, e6c654497e8749d3be0ea983b5f816e2=11421] 2024-11-10T11:01:21,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741889_1075 (size=7089) 2024-11-10T11:01:21,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741889_1075 (size=7089) 2024-11-10T11:01:21,544 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/info/ebd6752f5cbb464c880b4d300f3f87d5 2024-11-10T11:01:21,545 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/default/TestLogRolling-testLogRollOnDatanodeDeath/2cfec1cabbb1d72055901cb482dadc5a/recovered.edits/68.seqid, newMaxSeqId=68, maxSeqId=1 2024-11-10T11:01:21,546 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:21,546 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2cfec1cabbb1d72055901cb482dadc5a: Waiting for close lock at 1731236481499Running coprocessor pre-close hooks at 1731236481499Disabling compacts and flushes for region at 1731236481499Disabling writes for close at 1731236481499Obtaining lock to block concurrent updates at 1731236481500 (+1 ms)Preparing flush snapshotting stores in 2cfec1cabbb1d72055901cb482dadc5a at 1731236481500Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a., syncing WAL and waiting on mvcc, flushsize=dataSize=6455, getHeapSize=7152, getOffHeapSize=0, getCellsCount=6 at 1731236481500Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. at 1731236481501 (+1 ms)Flushing 2cfec1cabbb1d72055901cb482dadc5a/info: creating writer at 1731236481501Flushing 2cfec1cabbb1d72055901cb482dadc5a/info: appending metadata at 1731236481505 (+4 ms)Flushing 2cfec1cabbb1d72055901cb482dadc5a/info: closing flushed file at 1731236481505Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@328073f0: reopening flushed file at 1731236481520 (+15 ms)Finished flush of dataSize ~6.30 KB/6455, heapSize ~6.98 KB/7152, currentSize=0 B/0 for 2cfec1cabbb1d72055901cb482dadc5a in 29ms, sequenceid=65, compaction requested=false at 1731236481528 (+8 ms)Writing region close event to WAL at 1731236481541 (+13 ms)Running coprocessor post-close hooks at 1731236481546 (+5 ms)Closed at 1731236481546 2024-11-10T11:01:21,546 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1731236444724.2cfec1cabbb1d72055901cb482dadc5a. 2024-11-10T11:01:21,565 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/ns/9debc442df7f473fb25e2b7d47e5e630 is 43, key is default/ns:d/1731236444569/Put/seqid=0 2024-11-10T11:01:21,568 WARN [Thread-1060 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1076 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,567 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46836 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741890_1076] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6]'}, localName='127.0.0.1:43619', datanodeUuid='2c9c3a51-2087-4de6-a269-0c69b1bcc5c1', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741890_1076 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,568 WARN [Thread-1060 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741890_1076 in pipeline [DatanodeInfoWithStorage[127.0.0.1:43619,DS-906c57e6-b836-458c-ab59-619e9a182ef7,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,568 WARN [Thread-1060 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741890_1076 2024-11-10T11:01:21,568 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46836 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741890_1076] {}] datanode.BlockReceiver(316): Block 1073741890 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:21,568 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:46836 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741890_1076] {}] datanode.DataXceiver(331): 127.0.0.1:43619:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46836 dst: /127.0.0.1:43619 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,568 WARN [Thread-1060 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741891_1077 (size=5153) 2024-11-10T11:01:21,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741891_1077 (size=5153) 2024-11-10T11:01:21,573 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/ns/9debc442df7f473fb25e2b7d47e5e630 2024-11-10T11:01:21,594 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/table/265106098cb442c98a88b622ac2670dc is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1731236445094/Put/seqid=0 2024-11-10T11:01:21,596 WARN [Thread-1067 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1078 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34873 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:21,596 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:43454 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741892_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data4]'}, localName='127.0.0.1:45719', datanodeUuid='1fef1aa5-b235-4cbe-98ce-a6a9de5795ad', xmitsInProgress=0}:Exception transferring block BP-1123061454-172.17.0.2-1731236442906:blk_1073741892_1078 to mirror 127.0.0.1:34873 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,596 WARN [Thread-1067 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1123061454-172.17.0.2-1731236442906:blk_1073741892_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45719,DS-4e37e353-e3be-40ea-b636-273dd600cc83,DISK], DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK]) is bad. 2024-11-10T11:01:21,596 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:43454 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741892_1078] {}] datanode.BlockReceiver(316): Block 1073741892 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-10T11:01:21,596 WARN [Thread-1067 {}] hdfs.DataStreamer(1850): Abandoning BP-1123061454-172.17.0.2-1731236442906:blk_1073741892_1078 2024-11-10T11:01:21,596 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_627445597_22 at /127.0.0.1:43454 [Receiving block BP-1123061454-172.17.0.2-1731236442906:blk_1073741892_1078] {}] datanode.DataXceiver(331): 127.0.0.1:45719:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43454 dst: /127.0.0.1:45719 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:21,597 WARN [Thread-1067 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34873,DS-ec87db15-bdb7-4c78-8830-7c14672078ab,DISK] 2024-11-10T11:01:21,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741893_1079 (size=5424) 2024-11-10T11:01:21,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741893_1079 (size=5424) 2024-11-10T11:01:21,601 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/table/265106098cb442c98a88b622ac2670dc 2024-11-10T11:01:21,607 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/info/ebd6752f5cbb464c880b4d300f3f87d5 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/info/ebd6752f5cbb464c880b4d300f3f87d5 2024-11-10T11:01:21,612 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/info/ebd6752f5cbb464c880b4d300f3f87d5, entries=10, sequenceid=11, filesize=6.9 K 2024-11-10T11:01:21,613 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/ns/9debc442df7f473fb25e2b7d47e5e630 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/ns/9debc442df7f473fb25e2b7d47e5e630 2024-11-10T11:01:21,618 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/ns/9debc442df7f473fb25e2b7d47e5e630, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T11:01:21,619 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/.tmp/table/265106098cb442c98a88b622ac2670dc as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/table/265106098cb442c98a88b622ac2670dc 2024-11-10T11:01:21,626 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/table/265106098cb442c98a88b622ac2670dc, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T11:01:21,627 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=11, compaction requested=false 2024-11-10T11:01:21,634 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T11:01:21,635 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:01:21,635 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:01:21,635 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236481499Running coprocessor pre-close hooks at 1731236481499Disabling compacts and flushes for region at 1731236481499Disabling writes for close at 1731236481499Obtaining lock to block concurrent updates at 1731236481500 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731236481500Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1731236481500Flushing stores of hbase:meta,,1.1588230740 at 1731236481515 (+15 ms)Flushing 1588230740/info: creating writer at 1731236481516 (+1 ms)Flushing 1588230740/info: appending metadata at 1731236481534 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731236481534Flushing 1588230740/ns: creating writer at 1731236481550 (+16 ms)Flushing 1588230740/ns: appending metadata at 1731236481564 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731236481564Flushing 1588230740/table: creating writer at 1731236481578 (+14 ms)Flushing 1588230740/table: appending metadata at 1731236481593 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731236481593Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b52e075: reopening flushed file at 1731236481606 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b633617: reopening flushed file at 1731236481612 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f541ef4: reopening flushed file at 1731236481618 (+6 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 128ms, sequenceid=11, compaction requested=false at 1731236481628 (+10 ms)Writing region close event to WAL at 1731236481630 (+2 ms)Running coprocessor post-close hooks at 1731236481635 (+5 ms)Closed at 1731236481635 2024-11-10T11:01:21,635 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:01:21,674 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.1731236463466 to hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs/eccb6f743e07%2C46705%2C1731236443672.1731236463466 2024-11-10T11:01:21,699 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,46705,1731236443672; all regions closed. 2024-11-10T11:01:21,700 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,700 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,700 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,700 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,700 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:21,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741886_1071 (size=825) 2024-11-10T11:01:21,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741886_1071 (size=825) 2024-11-10T11:01:21,733 INFO [regionserver/eccb6f743e07:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T11:01:21,733 INFO [regionserver/eccb6f743e07:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T11:01:21,936 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:01:21,942 INFO [regionserver/eccb6f743e07:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T11:01:21,942 INFO [regionserver/eccb6f743e07:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T11:01:22,692 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:01:22,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:01:22,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:01:23,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741861_1044 (size=13591) 2024-11-10T11:01:23,834 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@522e5b85[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:45719, datanodeUuid=1fef1aa5-b235-4cbe-98ce-a6a9de5795ad, infoPort=46591, infoSecurePort=0, ipcPort=37265, storageInfo=lv=-57;cid=testClusterID;nsid=1588177168;c=1731236442906):Failed to transfer BP-1123061454-172.17.0.2-1731236442906:blk_1073741826_1002 to 127.0.0.1:34873 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:24,843 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T11:01:24,843 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T11:01:24,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-10T11:01:24,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:01:24,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:01:25,502 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 after 4002ms 2024-11-10T11:01:25,516 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta after 4002ms 2024-11-10T11:01:25,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741836_1012 (size=76) 2024-11-10T11:01:25,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:01:25,859 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@2277c405 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1123061454-172.17.0.2-1731236442906:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:34089,null,null]) java.net.ConnectException: Call From eccb6f743e07/172.17.0.2 to localhost:34949 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T11:01:26,500 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-10T11:01:26,502 DEBUG [RS:1;eccb6f743e07:41543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs 2024-11-10T11:01:26,502 INFO [RS:1;eccb6f743e07:41543 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C41543%2C1731236444637:(num 1731236444825) 2024-11-10T11:01:26,502 DEBUG [RS:1;eccb6f743e07:41543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:26,502 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:01:26,502 INFO [RS:1;eccb6f743e07:41543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:01:26,502 INFO [RS:1;eccb6f743e07:41543 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T11:01:26,503 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:01:26,503 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:01:26,503 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:01:26,503 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:01:26,503 INFO [RS:1;eccb6f743e07:41543 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:01:26,503 INFO [RS:1;eccb6f743e07:41543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41543 2024-11-10T11:01:26,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:01:26,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,41543,1731236444637 2024-11-10T11:01:26,505 INFO [RS:1;eccb6f743e07:41543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:01:26,506 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,41543,1731236444637] 2024-11-10T11:01:26,507 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,41543,1731236444637 already deleted, retry=false 2024-11-10T11:01:26,507 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,41543,1731236444637 expired; onlineServers=1 2024-11-10T11:01:26,507 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:26,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,561 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,561 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,561 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,561 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,562 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,569 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,570 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:26,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:26,606 INFO [RS:1;eccb6f743e07:41543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:01:26,606 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1010189a0650002, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:26,606 INFO [RS:1;eccb6f743e07:41543 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,41543,1731236444637; zookeeper connection closed. 2024-11-10T11:01:26,607 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3184a107 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3184a107 2024-11-10T11:01:26,701 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-10T11:01:26,704 DEBUG [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs 2024-11-10T11:01:26,704 INFO [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C46705%2C1731236443672.meta:.meta(num 1731236481500) 2024-11-10T11:01:26,704 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,705 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,705 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,705 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,705 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741880_1064 (size=15140) 2024-11-10T11:01:26,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741880_1064 (size=15140) 2024-11-10T11:01:26,710 DEBUG [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/oldWALs 2024-11-10T11:01:26,710 INFO [RS:0;eccb6f743e07:46705 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C46705%2C1731236443672:(num 1731236481262) 2024-11-10T11:01:26,710 DEBUG [RS:0;eccb6f743e07:46705 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:26,710 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:01:26,710 INFO [RS:0;eccb6f743e07:46705 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:01:26,710 INFO [RS:0;eccb6f743e07:46705 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T11:01:26,710 INFO [RS:0;eccb6f743e07:46705 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:01:26,710 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:01:26,710 INFO [RS:0;eccb6f743e07:46705 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46705 2024-11-10T11:01:26,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,46705,1731236443672 2024-11-10T11:01:26,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:01:26,714 INFO [RS:0;eccb6f743e07:46705 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:01:26,715 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,46705,1731236443672] 2024-11-10T11:01:26,716 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,46705,1731236443672 already deleted, retry=false 2024-11-10T11:01:26,716 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,46705,1731236443672 expired; onlineServers=0 2024-11-10T11:01:26,716 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,35673,1731236443623' ***** 2024-11-10T11:01:26,716 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:01:26,716 INFO [M:0;eccb6f743e07:35673 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:01:26,716 INFO [M:0;eccb6f743e07:35673 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:01:26,717 DEBUG [M:0;eccb6f743e07:35673 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:01:26,717 DEBUG [M:0;eccb6f743e07:35673 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:01:26,717 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:01:26,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236443884 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236443884,5,FailOnTimeoutGroup] 2024-11-10T11:01:26,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236443884 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236443884,5,FailOnTimeoutGroup] 2024-11-10T11:01:26,717 INFO [M:0;eccb6f743e07:35673 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:01:26,717 INFO [M:0;eccb6f743e07:35673 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:01:26,717 DEBUG [M:0;eccb6f743e07:35673 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:01:26,717 INFO [M:0;eccb6f743e07:35673 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:01:26,717 INFO [M:0;eccb6f743e07:35673 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:01:26,717 INFO [M:0;eccb6f743e07:35673 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:01:26,718 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:01:26,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:01:26,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:26,718 DEBUG [M:0;eccb6f743e07:35673 {}] zookeeper.ZKUtil(347): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:01:26,718 WARN [M:0;eccb6f743e07:35673 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:01:26,719 INFO [M:0;eccb6f743e07:35673 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/.lastflushedseqids 2024-11-10T11:01:26,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741894_1080 (size=130) 2024-11-10T11:01:26,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741894_1080 (size=130) 2024-11-10T11:01:26,725 INFO [M:0;eccb6f743e07:35673 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:01:26,725 INFO [M:0;eccb6f743e07:35673 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:01:26,725 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:01:26,725 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:26,726 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:26,726 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:01:26,726 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:26,726 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB 2024-11-10T11:01:26,744 DEBUG [M:0;eccb6f743e07:35673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e8516b20ad3a40e3830e4d45b28be8f9 is 82, key is hbase:meta,,1/info:regioninfo/1731236444552/Put/seqid=0 2024-11-10T11:01:26,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741895_1081 (size=5672) 2024-11-10T11:01:26,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741895_1081 (size=5672) 2024-11-10T11:01:26,750 INFO [M:0;eccb6f743e07:35673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e8516b20ad3a40e3830e4d45b28be8f9 2024-11-10T11:01:26,774 DEBUG [M:0;eccb6f743e07:35673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74c781159630439e83b58f7d76b5a9f3 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731236445099/Put/seqid=0 2024-11-10T11:01:26,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741896_1082 (size=6255) 2024-11-10T11:01:26,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741896_1082 (size=6255) 2024-11-10T11:01:26,780 INFO [M:0;eccb6f743e07:35673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74c781159630439e83b58f7d76b5a9f3 2024-11-10T11:01:26,785 INFO [M:0;eccb6f743e07:35673 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74c781159630439e83b58f7d76b5a9f3 2024-11-10T11:01:26,801 DEBUG [M:0;eccb6f743e07:35673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/98013c9745e14e05a03e85cb2207fcfa is 69, key is eccb6f743e07,41543,1731236444637/rs:state/1731236444679/Put/seqid=0 2024-11-10T11:01:26,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741897_1083 (size=5224) 2024-11-10T11:01:26,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741897_1083 (size=5224) 2024-11-10T11:01:26,806 INFO [M:0;eccb6f743e07:35673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/98013c9745e14e05a03e85cb2207fcfa 2024-11-10T11:01:26,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:26,815 INFO [RS:0;eccb6f743e07:46705 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:01:26,815 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46705-0x1010189a0650001, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:26,815 INFO [RS:0;eccb6f743e07:46705 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,46705,1731236443672; zookeeper connection closed. 2024-11-10T11:01:26,816 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b29dc54 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b29dc54 2024-11-10T11:01:26,816 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-10T11:01:26,827 DEBUG [M:0;eccb6f743e07:35673 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6c50990fa3674cd7804d5f4a0f89904c is 52, key is load_balancer_on/state:d/1731236444620/Put/seqid=0 2024-11-10T11:01:26,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741898_1084 (size=5056) 2024-11-10T11:01:26,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741898_1084 (size=5056) 2024-11-10T11:01:26,833 INFO [M:0;eccb6f743e07:35673 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6c50990fa3674cd7804d5f4a0f89904c 2024-11-10T11:01:26,839 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e8516b20ad3a40e3830e4d45b28be8f9 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e8516b20ad3a40e3830e4d45b28be8f9 2024-11-10T11:01:26,844 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e8516b20ad3a40e3830e4d45b28be8f9, entries=8, sequenceid=60, filesize=5.5 K 2024-11-10T11:01:26,845 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74c781159630439e83b58f7d76b5a9f3 as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74c781159630439e83b58f7d76b5a9f3 2024-11-10T11:01:26,851 INFO [M:0;eccb6f743e07:35673 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74c781159630439e83b58f7d76b5a9f3 2024-11-10T11:01:26,851 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74c781159630439e83b58f7d76b5a9f3, entries=6, sequenceid=60, filesize=6.1 K 2024-11-10T11:01:26,852 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/98013c9745e14e05a03e85cb2207fcfa as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/98013c9745e14e05a03e85cb2207fcfa 2024-11-10T11:01:26,858 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/98013c9745e14e05a03e85cb2207fcfa, entries=2, sequenceid=60, filesize=5.1 K 2024-11-10T11:01:26,859 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/6c50990fa3674cd7804d5f4a0f89904c as hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6c50990fa3674cd7804d5f4a0f89904c 2024-11-10T11:01:26,864 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/6c50990fa3674cd7804d5f4a0f89904c, entries=1, sequenceid=60, filesize=4.9 K 2024-11-10T11:01:26,865 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=60, compaction requested=false 2024-11-10T11:01:26,867 INFO [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:26,867 DEBUG [M:0;eccb6f743e07:35673 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236486725Disabling compacts and flushes for region at 1731236486725Disabling writes for close at 1731236486726 (+1 ms)Obtaining lock to block concurrent updates at 1731236486726Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236486726Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1731236486726Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236486727 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236486727Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236486743 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236486743Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236486756 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236486773 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236486773Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236486785 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236486801 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236486801Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236486812 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236486826 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236486826Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7431c7ee: reopening flushed file at 1731236486839 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1894b8d9: reopening flushed file at 1731236486845 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@788db2e9: reopening flushed file at 1731236486852 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65ecc83e: reopening flushed file at 1731236486858 (+6 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=60, compaction requested=false at 1731236486865 (+7 ms)Writing region close event to WAL at 1731236486866 (+1 ms)Closed at 1731236486866 2024-11-10T11:01:26,867 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,867 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,867 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,868 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,868 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:26,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43619 is added to blk_1073741878_1061 (size=1045) 2024-11-10T11:01:26,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45719 is added to blk_1073741878_1061 (size=1045) 2024-11-10T11:01:26,870 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:01:26,870 INFO [M:0;eccb6f743e07:35673 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:01:26,871 INFO [M:0;eccb6f743e07:35673 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35673 2024-11-10T11:01:26,871 INFO [M:0;eccb6f743e07:35673 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:01:26,977 INFO [M:0;eccb6f743e07:35673 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:01:26,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:26,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35673-0x1010189a0650000, quorum=127.0.0.1:54989, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:26,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2656efd1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:26,980 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@743041f4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:26,980 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:26,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@616b5398{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:26,980 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@798caa79{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:26,982 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:26,982 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:26,982 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1123061454-172.17.0.2-1731236442906 (Datanode Uuid 1fef1aa5-b235-4cbe-98ce-a6a9de5795ad) service to localhost/127.0.0.1:39141 2024-11-10T11:01:26,982 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:26,982 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data3/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:26,983 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data4/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:26,981 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f631c94 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:34089,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:34949 , LocalHost:localPort eccb6f743e07/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-10T11:01:26,983 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:26,983 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f631c94 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1123061454-172.17.0.2-1731236442906:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:45719,null,null], DatanodeInfoWithStorage[127.0.0.1:34089,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: No block pool offer service for bpid=BP-1123061454-172.17.0.2-1731236442906 2024-11-10T11:01:26,983 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f631c94 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:45719,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1123061454-172.17.0.2-1731236442906 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:26,983 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f631c94 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:34089,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1123061454-172.17.0.2-1731236442906 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:26,984 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@1f631c94 {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:45719,null,null], DatanodeInfoWithStorage[127.0.0.1:34089,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1123061454-172.17.0.2-1731236442906:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:45719,null,null], DatanodeInfoWithStorage[127.0.0.1:34089,null,null]] 2024-11-10T11:01:26,985 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7a2a3a4f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:26,985 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7fa27241{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:26,985 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:26,986 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@402062d6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:26,986 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b53e4f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:26,987 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:26,987 WARN [BP-1123061454-172.17.0.2-1731236442906 heartbeating to localhost/127.0.0.1:39141 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1123061454-172.17.0.2-1731236442906 (Datanode Uuid 2c9c3a51-2087-4de6-a269-0c69b1bcc5c1) service to localhost/127.0.0.1:39141 2024-11-10T11:01:26,987 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:26,987 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:26,988 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data5/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:26,988 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/cluster_4cf00964-40f2-06cd-b5fe-bdff933c0450/data/data6/current/BP-1123061454-172.17.0.2-1731236442906 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:26,988 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:26,994 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cf515b1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:01:26,994 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@c6abea1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:26,994 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:26,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5e7025d4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:26,995 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16369da1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:27,003 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:01:27,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:01:27,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=153 (was 78) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39141 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39141 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39141 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-15-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:38271 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:38271 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39141 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:39141 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39141 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39141 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:39141 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f9d8cbefdc8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-14-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007f9d8cbefdc8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-15-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:39141 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39141 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39141 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=104 (was 36) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8865 (was 9916) 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=153, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=104, ProcessCount=11, AvailableMemoryMB=8865 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.log.dir so I do NOT create it in target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/4c45bfaf-1c28-0fa4-7d2d-c098b6caa177/hadoop.tmp.dir so I do NOT create it in target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490, deleteOnExit=true 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/test.cache.data in system properties and HBase conf 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir in system properties and HBase conf 2024-11-10T11:01:27,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T11:01:27,054 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:01:27,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T11:01:27,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/nfs.dump.dir in system properties and HBase conf 2024-11-10T11:01:27,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir in system properties and HBase conf 2024-11-10T11:01:27,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:01:27,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T11:01:27,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T11:01:27,068 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:01:27,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:01:27,091 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,092 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,092 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,092 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,092 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,093 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,096 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,096 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,096 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,098 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:27,162 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:27,167 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:27,168 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:27,168 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:27,168 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:01:27,169 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:27,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a3130af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:27,169 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6dbadaea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:27,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@65e5bfc{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-33493-hadoop-hdfs-3_4_1-tests_jar-_-any-631537449421142846/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:01:27,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1c23e5e9{HTTP/1.1, (http/1.1)}{localhost:33493} 2024-11-10T11:01:27,286 INFO [Time-limited test {}] server.Server(415): Started @147857ms 2024-11-10T11:01:27,299 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:01:27,429 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:27,432 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:27,433 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:27,433 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:27,433 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:01:27,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31880142{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:27,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3de05dd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:27,509 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:27,517 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:27,549 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7fd8c23{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-40193-hadoop-hdfs-3_4_1-tests_jar-_-any-7191200728761239608/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:27,550 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@77037455{HTTP/1.1, (http/1.1)}{localhost:40193} 2024-11-10T11:01:27,550 INFO [Time-limited test {}] server.Server(415): Started @148121ms 2024-11-10T11:01:27,552 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:27,583 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:27,588 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:27,589 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:27,589 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:27,589 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:01:27,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@624b3986{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:27,590 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@179d1ca6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:27,645 WARN [Thread-1188 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data1/current/BP-831653338-172.17.0.2-1731236487110/current, will proceed with Du for space computation calculation, 2024-11-10T11:01:27,645 WARN [Thread-1189 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data2/current/BP-831653338-172.17.0.2-1731236487110/current, will proceed with Du for space computation calculation, 2024-11-10T11:01:27,666 WARN [Thread-1167 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:27,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfe7a74dfb5eb69d5 with lease ID 0x4c38dee71a3f46b7: Processing first storage report for DS-dc24f1e2-4926-43c6-a645-b610a5df7533 from datanode DatanodeRegistration(127.0.0.1:40115, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=37915, infoSecurePort=0, ipcPort=42773, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110) 2024-11-10T11:01:27,668 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe7a74dfb5eb69d5 with lease ID 0x4c38dee71a3f46b7: from storage DS-dc24f1e2-4926-43c6-a645-b610a5df7533 node DatanodeRegistration(127.0.0.1:40115, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=37915, infoSecurePort=0, ipcPort=42773, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:27,668 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfe7a74dfb5eb69d5 with lease ID 0x4c38dee71a3f46b7: Processing first storage report for DS-8d974b6a-7287-43ee-a8e0-cca6a1f52e85 from datanode DatanodeRegistration(127.0.0.1:40115, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=37915, infoSecurePort=0, ipcPort=42773, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110) 2024-11-10T11:01:27,668 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfe7a74dfb5eb69d5 with lease ID 0x4c38dee71a3f46b7: from storage DS-8d974b6a-7287-43ee-a8e0-cca6a1f52e85 node DatanodeRegistration(127.0.0.1:40115, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=37915, infoSecurePort=0, ipcPort=42773, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:27,709 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1a9fc515{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-46041-hadoop-hdfs-3_4_1-tests_jar-_-any-5231636540199895856/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:27,709 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@38a385f1{HTTP/1.1, (http/1.1)}{localhost:46041} 2024-11-10T11:01:27,710 INFO [Time-limited test {}] server.Server(415): Started @148281ms 2024-11-10T11:01:27,711 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:27,821 WARN [Thread-1214 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data3/current/BP-831653338-172.17.0.2-1731236487110/current, will proceed with Du for space computation calculation, 2024-11-10T11:01:27,821 WARN [Thread-1215 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data4/current/BP-831653338-172.17.0.2-1731236487110/current, will proceed with Du for space computation calculation, 2024-11-10T11:01:27,844 WARN [Thread-1203 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x143ad226d4a321d2 with lease ID 0x4c38dee71a3f46b8: Processing first storage report for DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975 from datanode DatanodeRegistration(127.0.0.1:45939, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=37937, infoSecurePort=0, ipcPort=40837, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110) 2024-11-10T11:01:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x143ad226d4a321d2 with lease ID 0x4c38dee71a3f46b8: from storage DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975 node DatanodeRegistration(127.0.0.1:45939, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=37937, infoSecurePort=0, ipcPort=40837, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T11:01:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x143ad226d4a321d2 with lease ID 0x4c38dee71a3f46b8: Processing first storage report for DS-737fba10-a04b-411e-a83f-42a81e14f110 from datanode DatanodeRegistration(127.0.0.1:45939, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=37937, infoSecurePort=0, ipcPort=40837, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110) 2024-11-10T11:01:27,847 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x143ad226d4a321d2 with lease ID 0x4c38dee71a3f46b8: from storage DS-737fba10-a04b-411e-a83f-42a81e14f110 node DatanodeRegistration(127.0.0.1:45939, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=37937, infoSecurePort=0, ipcPort=40837, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:27,936 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96 2024-11-10T11:01:27,939 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/zookeeper_0, clientPort=52878, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T11:01:27,940 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52878 2024-11-10T11:01:27,940 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:27,942 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:27,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:01:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:01:27,952 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8 with version=8 2024-11-10T11:01:27,952 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T11:01:27,954 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T11:01:27,954 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:01:27,955 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44595 2024-11-10T11:01:27,956 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44595 connecting to ZooKeeper ensemble=127.0.0.1:52878 2024-11-10T11:01:27,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:445950x0, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:01:27,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44595-0x101018a4d920000 connected 2024-11-10T11:01:27,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:27,983 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:27,985 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:27,985 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8, hbase.cluster.distributed=false 2024-11-10T11:01:27,986 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:01:27,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44595 2024-11-10T11:01:27,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44595 2024-11-10T11:01:27,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44595 2024-11-10T11:01:27,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44595 2024-11-10T11:01:27,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44595 2024-11-10T11:01:28,002 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:01:28,002 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:01:28,002 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:01:28,002 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:01:28,003 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:01:28,003 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:01:28,003 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:01:28,003 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:01:28,003 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45733 2024-11-10T11:01:28,005 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45733 connecting to ZooKeeper ensemble=127.0.0.1:52878 2024-11-10T11:01:28,005 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:28,007 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:28,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457330x0, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:01:28,011 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:457330x0, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:28,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45733-0x101018a4d920001 connected 2024-11-10T11:01:28,011 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:01:28,012 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:01:28,012 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T11:01:28,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:01:28,013 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45733 2024-11-10T11:01:28,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45733 2024-11-10T11:01:28,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45733 2024-11-10T11:01:28,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45733 2024-11-10T11:01:28,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45733 2024-11-10T11:01:28,026 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:44595 2024-11-10T11:01:28,027 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:01:28,029 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:01:28,029 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T11:01:28,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,031 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:01:28,031 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,44595,1731236487953 from backup master directory 2024-11-10T11:01:28,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:01:28,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:01:28,034 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:01:28,034 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,038 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/hbase.id] with ID: 5d2b034c-e94e-4255-b239-a6d60091ce77 2024-11-10T11:01:28,039 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/.tmp/hbase.id 2024-11-10T11:01:28,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:01:28,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:01:28,047 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/.tmp/hbase.id]:[hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/hbase.id] 2024-11-10T11:01:28,059 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:28,059 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T11:01:28,060 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T11:01:28,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:01:28,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:01:28,070 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:01:28,070 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T11:01:28,071 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:01:28,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:01:28,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:01:28,083 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store 2024-11-10T11:01:28,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:01:28,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:01:28,092 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:01:28,092 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:01:28,092 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:28,092 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:28,092 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:01:28,092 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:28,092 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:28,092 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236488092Disabling compacts and flushes for region at 1731236488092Disabling writes for close at 1731236488092Writing region close event to WAL at 1731236488092Closed at 1731236488092 2024-11-10T11:01:28,093 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/.initializing 2024-11-10T11:01:28,093 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,096 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C44595%2C1731236487953, suffix=, logDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953, archiveDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/oldWALs, maxLogs=10 2024-11-10T11:01:28,097 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C44595%2C1731236487953.1731236488097 2024-11-10T11:01:28,105 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 2024-11-10T11:01:28,106 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37937:37937),(127.0.0.1/127.0.0.1:37915:37915)] 2024-11-10T11:01:28,107 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:01:28,107 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:01:28,107 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,107 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T11:01:28,110 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,111 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T11:01:28,111 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:01:28,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T11:01:28,113 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:01:28,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T11:01:28,115 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:01:28,116 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,116 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,116 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,118 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,118 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,118 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T11:01:28,119 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:01:28,122 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:01:28,122 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=771467, jitterRate=-0.019029855728149414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T11:01:28,123 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236488107Initializing all the Stores at 1731236488108 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488108Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236488108Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236488108Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236488108Cleaning up temporary data from old regions at 1731236488118 (+10 ms)Region opened successfully at 1731236488123 (+5 ms) 2024-11-10T11:01:28,124 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T11:01:28,128 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6781557d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:01:28,129 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T11:01:28,129 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T11:01:28,129 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T11:01:28,129 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T11:01:28,130 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T11:01:28,130 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T11:01:28,130 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T11:01:28,132 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T11:01:28,133 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T11:01:28,134 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T11:01:28,135 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T11:01:28,135 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T11:01:28,137 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T11:01:28,137 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T11:01:28,138 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T11:01:28,140 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T11:01:28,142 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T11:01:28,143 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T11:01:28,146 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T11:01:28,147 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T11:01:28,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:28,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:28,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,150 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,44595,1731236487953, sessionid=0x101018a4d920000, setting cluster-up flag (Was=false) 2024-11-10T11:01:28,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,160 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T11:01:28,161 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,171 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T11:01:28,172 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,44595,1731236487953 2024-11-10T11:01:28,173 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T11:01:28,175 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T11:01:28,175 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T11:01:28,176 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T11:01:28,176 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,44595,1731236487953 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:01:28,177 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236518178 2024-11-10T11:01:28,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T11:01:28,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T11:01:28,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T11:01:28,178 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T11:01:28,179 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T11:01:28,179 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T11:01:28,179 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,179 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T11:01:28,179 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:01:28,179 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T11:01:28,179 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T11:01:28,179 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T11:01:28,180 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T11:01:28,180 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T11:01:28,180 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236488180,5,FailOnTimeoutGroup] 2024-11-10T11:01:28,180 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236488180,5,FailOnTimeoutGroup] 2024-11-10T11:01:28,180 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,180 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T11:01:28,180 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,180 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,180 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,180 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T11:01:28,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:01:28,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:01:28,216 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(746): ClusterId : 5d2b034c-e94e-4255-b239-a6d60091ce77 2024-11-10T11:01:28,216 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:01:28,219 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:01:28,219 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:01:28,221 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:01:28,221 DEBUG [RS:0;eccb6f743e07:45733 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffef769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:01:28,233 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:45733 2024-11-10T11:01:28,234 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:01:28,234 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:01:28,234 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:01:28,234 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,44595,1731236487953 with port=45733, startcode=1731236488002 2024-11-10T11:01:28,235 DEBUG [RS:0;eccb6f743e07:45733 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:01:28,237 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52475, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:01:28,237 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44595 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,237 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44595 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,239 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8 2024-11-10T11:01:28,239 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43083 2024-11-10T11:01:28,239 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:01:28,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:01:28,242 DEBUG [RS:0;eccb6f743e07:45733 {}] zookeeper.ZKUtil(111): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,242 WARN [RS:0;eccb6f743e07:45733 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:01:28,243 INFO [RS:0;eccb6f743e07:45733 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:01:28,243 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,243 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,45733,1731236488002] 2024-11-10T11:01:28,246 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:01:28,248 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:01:28,248 INFO [RS:0;eccb6f743e07:45733 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:01:28,248 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,252 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:01:28,253 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:01:28,253 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,253 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,253 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,253 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,253 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:01:28,254 DEBUG [RS:0;eccb6f743e07:45733 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:01:28,256 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,256 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,256 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,256 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,256 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,256 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45733,1731236488002-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:01:28,271 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:01:28,271 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45733,1731236488002-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,271 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,272 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.Replication(171): eccb6f743e07,45733,1731236488002 started 2024-11-10T11:01:28,286 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,286 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,45733,1731236488002, RpcServer on eccb6f743e07/172.17.0.2:45733, sessionid=0x101018a4d920001 2024-11-10T11:01:28,286 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:01:28,286 DEBUG [RS:0;eccb6f743e07:45733 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,286 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,45733,1731236488002' 2024-11-10T11:01:28,286 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,45733,1731236488002' 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:01:28,287 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:01:28,288 DEBUG [RS:0;eccb6f743e07:45733 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:01:28,288 INFO [RS:0;eccb6f743e07:45733 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:01:28,288 INFO [RS:0;eccb6f743e07:45733 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:01:28,390 INFO [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C45733%2C1731236488002, suffix=, logDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002, archiveDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs, maxLogs=32 2024-11-10T11:01:28,391 INFO [RS:0;eccb6f743e07:45733 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:28,397 INFO [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:28,398 DEBUG [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37915:37915),(127.0.0.1/127.0.0.1:37937:37937)] 2024-11-10T11:01:28,509 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:28,517 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:28,591 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T11:01:28,591 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8 2024-11-10T11:01:28,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741833_1009 (size=32) 2024-11-10T11:01:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741833_1009 (size=32) 2024-11-10T11:01:28,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:01:28,599 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:01:28,601 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:01:28,601 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:01:28,602 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:01:28,602 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:01:28,604 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:01:28,604 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:01:28,606 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:01:28,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:01:28,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740 2024-11-10T11:01:28,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740 2024-11-10T11:01:28,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:01:28,608 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:01:28,609 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:01:28,610 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:01:28,612 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:01:28,612 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=707420, jitterRate=-0.10047009587287903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:01:28,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236488598Initializing all the Stores at 1731236488599 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488599Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488599Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236488599Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488599Cleaning up temporary data from old regions at 1731236488608 (+9 ms)Region opened successfully at 1731236488613 (+5 ms) 2024-11-10T11:01:28,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:01:28,613 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:01:28,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:01:28,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:01:28,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:01:28,613 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:01:28,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236488613Disabling compacts and flushes for region at 1731236488613Disabling writes for close at 1731236488613Writing region close event to WAL at 1731236488613Closed at 1731236488613 2024-11-10T11:01:28,615 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:01:28,615 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T11:01:28,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T11:01:28,616 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:01:28,617 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T11:01:28,768 DEBUG [eccb6f743e07:44595 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T11:01:28,768 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,770 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,45733,1731236488002, state=OPENING 2024-11-10T11:01:28,771 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T11:01:28,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:28,773 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:01:28,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:01:28,773 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:01:28,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,45733,1731236488002}] 2024-11-10T11:01:28,927 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T11:01:28,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59351, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T11:01:28,932 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T11:01:28,932 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:01:28,934 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C45733%2C1731236488002.meta, suffix=.meta, logDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002, archiveDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs, maxLogs=32 2024-11-10T11:01:28,935 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta 2024-11-10T11:01:28,940 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta 2024-11-10T11:01:28,940 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37915:37915),(127.0.0.1/127.0.0.1:37937:37937)] 2024-11-10T11:01:28,941 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:01:28,941 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T11:01:28,941 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T11:01:28,942 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T11:01:28,942 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T11:01:28,942 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:01:28,942 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T11:01:28,942 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T11:01:28,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:01:28,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:01:28,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:01:28,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:01:28,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:01:28,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:01:28,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:01:28,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:01:28,948 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:28,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:01:28,948 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:01:28,949 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740 2024-11-10T11:01:28,950 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740 2024-11-10T11:01:28,951 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:01:28,951 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:01:28,951 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:01:28,952 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:01:28,953 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=750586, jitterRate=-0.04558061063289642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:01:28,953 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T11:01:28,954 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236488942Writing region info on filesystem at 1731236488942Initializing all the Stores at 1731236488943 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488943Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488943Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236488943Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236488943Cleaning up temporary data from old regions at 1731236488951 (+8 ms)Running coprocessor post-open hooks at 1731236488953 (+2 ms)Region opened successfully at 1731236488954 (+1 ms) 2024-11-10T11:01:28,955 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236488926 2024-11-10T11:01:28,958 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T11:01:28,958 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T11:01:28,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,960 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,45733,1731236488002, state=OPEN 2024-11-10T11:01:28,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:01:28,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:01:28,965 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,45733,1731236488002 2024-11-10T11:01:28,965 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:01:28,965 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:01:28,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T11:01:28,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,45733,1731236488002 in 192 msec 2024-11-10T11:01:28,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T11:01:28,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 353 msec 2024-11-10T11:01:28,971 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:01:28,971 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T11:01:28,972 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:01:28,972 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,45733,1731236488002, seqNum=-1] 2024-11-10T11:01:28,972 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:01:28,974 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55585, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:01:28,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 803 msec 2024-11-10T11:01:28,979 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236488979, completionTime=-1 2024-11-10T11:01:28,979 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T11:01:28,979 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236548981 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236608981 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,44595,1731236487953-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,44595,1731236487953-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,44595,1731236487953-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:44595, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,981 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,982 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:28,983 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.951sec 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,44595,1731236487953-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:01:28,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,44595,1731236487953-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T11:01:28,987 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T11:01:28,987 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T11:01:28,987 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,44595,1731236487953-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:01:29,017 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7f214b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:01:29,017 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,44595,-1 for getting cluster id 2024-11-10T11:01:29,017 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T11:01:29,019 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5d2b034c-e94e-4255-b239-a6d60091ce77' 2024-11-10T11:01:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T11:01:29,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5d2b034c-e94e-4255-b239-a6d60091ce77" 2024-11-10T11:01:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8da49e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:01:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,44595,-1] 2024-11-10T11:01:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T11:01:29,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:29,022 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T11:01:29,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40375d59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:01:29,023 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:01:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,45733,1731236488002, seqNum=-1] 2024-11-10T11:01:29,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:01:29,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:01:29,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,44595,1731236487953 2024-11-10T11:01:29,028 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:01:29,031 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T11:01:29,031 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-11-10T11:01:29,031 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-11-10T11:01:29,031 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T11:01:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is eccb6f743e07,44595,1731236487953 2024-11-10T11:01:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f78b918 2024-11-10T11:01:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T11:01:29,034 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T11:01:29,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T11:01:29,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T11:01:29,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:01:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T11:01:29,038 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T11:01:29,038 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:29,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-11-10T11:01:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:01:29,039 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T11:01:29,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741835_1011 (size=395) 2024-11-10T11:01:29,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741835_1011 (size=395) 2024-11-10T11:01:29,048 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33af8a6ffe292c16a5b1951425fba44a, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8 2024-11-10T11:01:29,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40115 is added to blk_1073741836_1012 (size=78) 2024-11-10T11:01:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45939 is added to blk_1073741836_1012 (size=78) 2024-11-10T11:01:29,054 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:01:29,054 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 33af8a6ffe292c16a5b1951425fba44a, disabling compactions & flushes 2024-11-10T11:01:29,054 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,054 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,054 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. after waiting 0 ms 2024-11-10T11:01:29,054 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,054 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,054 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33af8a6ffe292c16a5b1951425fba44a: Waiting for close lock at 1731236489054Disabling compacts and flushes for region at 1731236489054Disabling writes for close at 1731236489054Writing region close event to WAL at 1731236489054Closed at 1731236489054 2024-11-10T11:01:29,056 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T11:01:29,056 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1731236489056"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236489056"}]},"ts":"1731236489056"} 2024-11-10T11:01:29,059 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T11:01:29,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T11:01:29,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236489060"}]},"ts":"1731236489060"} 2024-11-10T11:01:29,062 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-11-10T11:01:29,062 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=33af8a6ffe292c16a5b1951425fba44a, ASSIGN}] 2024-11-10T11:01:29,064 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=33af8a6ffe292c16a5b1951425fba44a, ASSIGN 2024-11-10T11:01:29,065 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=33af8a6ffe292c16a5b1951425fba44a, ASSIGN; state=OFFLINE, location=eccb6f743e07,45733,1731236488002; forceNewPlan=false, retain=false 2024-11-10T11:01:29,216 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=33af8a6ffe292c16a5b1951425fba44a, regionState=OPENING, regionLocation=eccb6f743e07,45733,1731236488002 2024-11-10T11:01:29,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=33af8a6ffe292c16a5b1951425fba44a, ASSIGN because future has completed 2024-11-10T11:01:29,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33af8a6ffe292c16a5b1951425fba44a, server=eccb6f743e07,45733,1731236488002}] 2024-11-10T11:01:29,377 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,377 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 33af8a6ffe292c16a5b1951425fba44a, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:01:29,377 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,377 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:01:29,377 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,377 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,379 INFO [StoreOpener-33af8a6ffe292c16a5b1951425fba44a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,380 INFO [StoreOpener-33af8a6ffe292c16a5b1951425fba44a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33af8a6ffe292c16a5b1951425fba44a columnFamilyName info 2024-11-10T11:01:29,380 DEBUG [StoreOpener-33af8a6ffe292c16a5b1951425fba44a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:01:29,380 INFO [StoreOpener-33af8a6ffe292c16a5b1951425fba44a-1 {}] regionserver.HStore(327): Store=33af8a6ffe292c16a5b1951425fba44a/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:01:29,380 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,381 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,382 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,382 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,382 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,383 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,386 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:01:29,386 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 33af8a6ffe292c16a5b1951425fba44a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=869647, jitterRate=0.10581357777118683}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T11:01:29,386 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:29,387 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 33af8a6ffe292c16a5b1951425fba44a: Running coprocessor pre-open hook at 1731236489377Writing region info on filesystem at 1731236489377Initializing all the Stores at 1731236489378 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236489378Cleaning up temporary data from old regions at 1731236489382 (+4 ms)Running coprocessor post-open hooks at 1731236489386 (+4 ms)Region opened successfully at 1731236489387 (+1 ms) 2024-11-10T11:01:29,388 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a., pid=6, masterSystemTime=1731236489372 2024-11-10T11:01:29,390 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,390 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:29,391 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=33af8a6ffe292c16a5b1951425fba44a, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,45733,1731236488002 2024-11-10T11:01:29,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33af8a6ffe292c16a5b1951425fba44a, server=eccb6f743e07,45733,1731236488002 because future has completed 2024-11-10T11:01:29,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T11:01:29,398 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 33af8a6ffe292c16a5b1951425fba44a, server=eccb6f743e07,45733,1731236488002 in 176 msec 2024-11-10T11:01:29,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T11:01:29,401 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=33af8a6ffe292c16a5b1951425fba44a, ASSIGN in 336 msec 2024-11-10T11:01:29,402 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T11:01:29,402 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236489402"}]},"ts":"1731236489402"} 2024-11-10T11:01:29,404 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-11-10T11:01:29,405 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T11:01:29,407 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 370 msec 2024-11-10T11:01:29,510 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:29,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:30,511 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:30,518 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:31,511 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:31,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:32,512 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:32,519 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:33,513 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:33,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:34,444 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:01:34,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,458 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,459 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,460 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,463 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,463 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,464 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,466 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:34,471 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T11:01:34,472 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-11-10T11:01:34,513 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:34,520 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:34,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:01:34,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T11:01:34,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T11:01:34,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-11-10T11:01:34,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:01:34,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T11:01:35,514 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:35,521 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:36,515 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:36,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:37,515 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:37,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:38,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:38,522 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44595 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:01:39,127 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-11-10T11:01:39,127 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-11-10T11:01:39,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T11:01:39,130 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:39,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a., hostname=eccb6f743e07,45733,1731236488002, seqNum=2] 2024-11-10T11:01:39,516 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:39,523 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:40,517 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:40,523 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:41,136 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:41,137 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:41,137 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008 java.io.IOException: Bad response ERROR for BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008 from datanode DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:41,137 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:41,137 WARN [DataStreamer for file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 block BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK], DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]) is bad. 2024-11-10T11:01:41,138 WARN [DataStreamer for file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta block BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK], DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]) is bad. 2024-11-10T11:01:41,138 WARN [DataStreamer for file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 block BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008 {}] hdfs.DataStreamer(1731): Error Recovery for BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK], DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:45939,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]) is bad. 2024-11-10T11:01:41,138 WARN [PacketResponder: BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45939] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,138 WARN [PacketResponder: BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:45939] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1320211877_22 at /127.0.0.1:41670 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:45939:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41670 dst: /127.0.0.1:45939 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:48540 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48540 dst: /127.0.0.1:40115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:48518 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:40115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48518 dst: /127.0.0.1:40115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,138 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1320211877_22 at /127.0.0.1:48498 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48498 dst: /127.0.0.1:40115 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,139 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:41682 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:45939:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41682 dst: /127.0.0.1:45939 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,139 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:41708 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:45939:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41708 dst: /127.0.0.1:45939 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,140 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1a9fc515{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:41,140 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@38a385f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:41,140 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:41,141 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@179d1ca6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:41,141 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@624b3986{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:41,142 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:41,142 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-831653338-172.17.0.2-1731236487110 (Datanode Uuid 29328916-b262-4879-ae98-dfb609c1d371) service to localhost/127.0.0.1:43083 2024-11-10T11:01:41,142 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:41,143 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:41,143 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data3/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:41,143 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data4/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:41,143 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:41,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:41,155 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:41,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:41,156 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:41,156 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:01:41,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5507dc7e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:41,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27e9dc43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:41,272 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7aa94e72{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-45909-hadoop-hdfs-3_4_1-tests_jar-_-any-14388512681645571683/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:41,272 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@f09ac85{HTTP/1.1, (http/1.1)}{localhost:45909} 2024-11-10T11:01:41,272 INFO [Time-limited test {}] server.Server(415): Started @161843ms 2024-11-10T11:01:41,274 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:41,292 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:41,292 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:41,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:45238 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741832_1008] {}] datanode.DataXceiver(331): 127.0.0.1:40115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45238 dst: /127.0.0.1:40115 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,292 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:41,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1320211877_22 at /127.0.0.1:45252 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45252 dst: /127.0.0.1:40115 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,293 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:45236 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40115:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45236 dst: /127.0.0.1:40115 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:41,298 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7fd8c23{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:41,298 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@77037455{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:41,298 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:41,299 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3de05dd9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:41,299 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31880142{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:41,300 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:41,300 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-831653338-172.17.0.2-1731236487110 (Datanode Uuid 3a712815-02f0-4d5d-b159-846cbc068add) service to localhost/127.0.0.1:43083 2024-11-10T11:01:41,300 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:41,300 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:41,302 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data1/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:41,302 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data2/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:41,302 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:41,314 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:41,319 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:41,320 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:41,320 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:41,320 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:01:41,320 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a7a4b6c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:41,321 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f3a1cd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:41,366 WARN [Thread-1338 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd622550c2555bc76 with lease ID 0x4c38dee71a3f46b9: from storage DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975 node DatanodeRegistration(127.0.0.1:36561, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=44203, infoSecurePort=0, ipcPort=38377, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:41,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd622550c2555bc76 with lease ID 0x4c38dee71a3f46b9: from storage DS-737fba10-a04b-411e-a83f-42a81e14f110 node DatanodeRegistration(127.0.0.1:36561, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=44203, infoSecurePort=0, ipcPort=38377, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:41,440 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3903d405{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-44319-hadoop-hdfs-3_4_1-tests_jar-_-any-14380693735607094207/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:41,440 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3977d0c{HTTP/1.1, (http/1.1)}{localhost:44319} 2024-11-10T11:01:41,440 INFO [Time-limited test {}] server.Server(415): Started @162011ms 2024-11-10T11:01:41,442 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:41,549 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:41,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:41,571 WARN [Thread-1369 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:41,573 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x372e35ade054f5e1 with lease ID 0x4c38dee71a3f46ba: from storage DS-dc24f1e2-4926-43c6-a645-b610a5df7533 node DatanodeRegistration(127.0.0.1:41011, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=33991, infoSecurePort=0, ipcPort=42157, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:41,573 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x372e35ade054f5e1 with lease ID 0x4c38dee71a3f46ba: from storage DS-8d974b6a-7287-43ee-a8e0-cca6a1f52e85 node DatanodeRegistration(127.0.0.1:41011, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=33991, infoSecurePort=0, ipcPort=42157, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:42,459 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-11-10T11:01:42,462 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-11-10T11:01:42,463 ERROR [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:42,464 WARN [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:42,464 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C45733%2C1731236488002:(num 1731236488391) roll requested 2024-11-10T11:01:42,464 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:42,469 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 newFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:42,470 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:42,470 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:42,470 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:42,470 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:42,470 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:42,470 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:42,471 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:42,471 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:42,471 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:42,471 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33991:33991),(127.0.0.1/127.0.0.1:44203:44203)] 2024-11-10T11:01:42,471 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 is not closed yet, will try archiving it next time 2024-11-10T11:01:42,471 WARN [IPC Server handler 1 on default port 43083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741832_1013 2024-11-10T11:01:42,471 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 after 0ms 2024-11-10T11:01:42,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:42,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:43,550 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:43,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:44,475 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-11-10T11:01:44,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:44,551 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:45,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:45,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:46,369 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741832_1013: GenerationStamp not matched, existing replica is blk_1073741832_1008 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T11:01:46,472 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 after 4001ms 2024-11-10T11:01:46,477 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:46,478 WARN [DataStreamer for file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 block BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:41011,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK], DatanodeInfoWithStorage[127.0.0.1:36561,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:41011,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]) is bad. 2024-11-10T11:01:46,478 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:50506 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:41011:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50506 dst: /127.0.0.1:41011 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:46,478 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:56972 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36561:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56972 dst: /127.0.0.1:36561 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:46,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3903d405{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:46,481 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3977d0c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:46,481 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:46,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f3a1cd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:46,481 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a7a4b6c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:46,483 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:46,483 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:46,483 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:46,483 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-831653338-172.17.0.2-1731236487110 (Datanode Uuid 3a712815-02f0-4d5d-b159-846cbc068add) service to localhost/127.0.0.1:43083 2024-11-10T11:01:46,483 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data1/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:46,484 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data2/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:46,484 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:46,492 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:46,496 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:46,497 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:46,497 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:46,497 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:01:46,499 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@717499fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:46,499 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62802d30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:46,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:46,552 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:46,616 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1dcfabbb{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-46731-hadoop-hdfs-3_4_1-tests_jar-_-any-14135561273118719987/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:46,616 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6614137b{HTTP/1.1, (http/1.1)}{localhost:46731} 2024-11-10T11:01:46,617 INFO [Time-limited test {}] server.Server(415): Started @167187ms 2024-11-10T11:01:46,618 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:46,642 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:46,642 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1756358815_22 at /127.0.0.1:56996 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:36561:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56996 dst: /127.0.0.1:36561 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:46,645 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7aa94e72{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:46,645 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@f09ac85{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:01:46,646 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:01:46,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27e9dc43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:01:46,646 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5507dc7e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:01:46,647 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:01:46,647 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-831653338-172.17.0.2-1731236487110 (Datanode Uuid 29328916-b262-4879-ae98-dfb609c1d371) service to localhost/127.0.0.1:43083 2024-11-10T11:01:46,647 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:01:46,647 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:01:46,648 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data3/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:46,648 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data4/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:01:46,648 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:01:46,659 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:01:46,661 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:01:46,667 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:01:46,667 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:01:46,667 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:01:46,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3146549d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:01:46,668 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6baabd83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:01:46,713 WARN [Thread-1412 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:46,715 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a5cd0cba61cffea with lease ID 0x4c38dee71a3f46bb: from storage DS-dc24f1e2-4926-43c6-a645-b610a5df7533 node DatanodeRegistration(127.0.0.1:39429, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=40609, infoSecurePort=0, ipcPort=45223, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:46,716 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2a5cd0cba61cffea with lease ID 0x4c38dee71a3f46bb: from storage DS-8d974b6a-7287-43ee-a8e0-cca6a1f52e85 node DatanodeRegistration(127.0.0.1:39429, datanodeUuid=3a712815-02f0-4d5d-b159-846cbc068add, infoPort=40609, infoSecurePort=0, ipcPort=45223, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 7, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T11:01:46,785 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@21404da7{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/java.io.tmpdir/jetty-localhost-45161-hadoop-hdfs-3_4_1-tests_jar-_-any-16913432691861960562/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:01:46,785 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e8914c8{HTTP/1.1, (http/1.1)}{localhost:45161} 2024-11-10T11:01:46,785 INFO [Time-limited test {}] server.Server(415): Started @167356ms 2024-11-10T11:01:46,787 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:01:46,877 WARN [Thread-1443 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:01:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7844bd6f4bb2af0 with lease ID 0x4c38dee71a3f46bc: from storage DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975 node DatanodeRegistration(127.0.0.1:41935, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=33517, infoSecurePort=0, ipcPort=35093, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:46,880 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7844bd6f4bb2af0 with lease ID 0x4c38dee71a3f46bc: from storage DS-737fba10-a04b-411e-a83f-42a81e14f110 node DatanodeRegistration(127.0.0.1:41935, datanodeUuid=29328916-b262-4879-ae98-dfb609c1d371, infoPort=33517, infoSecurePort=0, ipcPort=35093, storageInfo=lv=-57;cid=testClusterID;nsid=2039527922;c=1731236487110), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:01:47,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:47,553 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:47,806 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-11-10T11:01:47,808 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-11-10T11:01:47,809 ERROR [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36561,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:47,809 WARN [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36561,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:47,810 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C45733%2C1731236488002:(num 1731236502464) roll requested 2024-11-10T11:01:47,810 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.1731236507810 2024-11-10T11:01:47,815 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 newFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 2024-11-10T11:01:47,815 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:47,815 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:47,815 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:47,816 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:47,816 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:47,816 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 2024-11-10T11:01:47,816 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36561,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:47,816 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:36561,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:47,816 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:47,816 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40609:40609),(127.0.0.1/127.0.0.1:33517:33517)] 2024-11-10T11:01:47,817 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 is not closed yet, will try archiving it next time 2024-11-10T11:01:47,817 WARN [IPC Server handler 4 on default port 43083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-11-10T11:01:47,817 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 after 1ms 2024-11-10T11:01:48,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:48,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:48,715 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T11:01:49,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:49,554 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:49,818 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:49,824 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 newFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:49,824 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:49,824 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:49,824 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:49,824 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:49,824 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:49,825 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:49,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40609:40609),(127.0.0.1/127.0.0.1:33517:33517)] 2024-11-10T11:01:49,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 is not closed yet, will try archiving it next time 2024-11-10T11:01:49,825 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 is not closed yet, will try archiving it next time 2024-11-10T11:01:49,826 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:49,826 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:49,826 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 after 0ms 2024-11-10T11:01:49,826 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:49,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741838_1019 (size=1264) 2024-11-10T11:01:49,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741838_1019 (size=1264) 2024-11-10T11:01:49,827 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 is not closed yet, will try archiving it next time 2024-11-10T11:01:49,835 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1731236489387/Put/vlen=218/seqid=0] 2024-11-10T11:01:49,835 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1731236499134/Put/vlen=1045/seqid=0] 2024-11-10T11:01:49,835 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236488391 2024-11-10T11:01:49,835 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:49,835 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:49,836 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 after 1ms 2024-11-10T11:01:49,836 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:49,839 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1731236502463/Put/vlen=1045/seqid=0] 2024-11-10T11:01:49,839 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1731236504476/Put/vlen=1045/seqid=0] 2024-11-10T11:01:49,839 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 2024-11-10T11:01:49,839 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 2024-11-10T11:01:49,839 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 2024-11-10T11:01:49,839 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 after 0ms 2024-11-10T11:01:49,839 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236507810 2024-11-10T11:01:49,842 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1731236507809/Put/vlen=1045/seqid=0] 2024-11-10T11:01:49,842 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:49,842 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:49,842 WARN [IPC Server handler 1 on default port 43083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-11-10T11:01:49,843 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 after 0ms 2024-11-10T11:01:50,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:50,555 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:50,716 WARN [ResponseProcessor for block BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:50,716 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1320211877_22 at /127.0.0.1:39218 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:39429:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39218 dst: /127.0.0.1:39429 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39429 remote=/127.0.0.1:39218]. Total timeout mills is 60000, 59108 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:50,716 WARN [DataStreamer for file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 block BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39429,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK], DatanodeInfoWithStorage[127.0.0.1:41935,DS-85c98deb-7aa6-4dec-9a9f-ec02b5b24975,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39429,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]) is bad. 2024-11-10T11:01:50,716 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1320211877_22 at /127.0.0.1:49808 [Receiving block BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:41935:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:49808 dst: /127.0.0.1:41935 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:01:50,717 WARN [DataStreamer for file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 block BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:50,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741839_1022 (size=85) 2024-11-10T11:01:50,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741839_1022 (size=85) 2024-11-10T11:01:51,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:51,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:51,818 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236502464 after 4002ms 2024-11-10T11:01:52,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:52,556 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:53,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:53,557 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:53,843 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 after 4001ms 2024-11-10T11:01:53,843 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:53,847 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:53,848 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-11-10T11:01:53,848 ERROR [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,848 WARN [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,848 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C45733%2C1731236488002.meta:.meta(num 1731236488934) roll requested 2024-11-10T11:01:53,849 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.meta.1731236513848.meta 2024-11-10T11:01:53,853 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,854 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,854 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,854 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,854 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,854 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236513848.meta 2024-11-10T11:01:53,854 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,855 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,855 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta 2024-11-10T11:01:53,855 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33517:33517),(127.0.0.1/127.0.0.1:40609:40609)] 2024-11-10T11:01:53,855 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta is not closed yet, will try archiving it next time 2024-11-10T11:01:53,855 WARN [IPC Server handler 3 on default port 43083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1015 2024-11-10T11:01:53,855 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta after 0ms 2024-11-10T11:01:53,871 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/info/795aa29f80e84a6db743382b11674930 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a./info:regioninfo/1731236489391/Put/seqid=0 2024-11-10T11:01:53,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741841_1025 (size=7125) 2024-11-10T11:01:53,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741841_1025 (size=7125) 2024-11-10T11:01:53,876 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/info/795aa29f80e84a6db743382b11674930 2024-11-10T11:01:53,895 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/ns/2ab74c65b8704872bed7f0dcfb567534 is 43, key is default/ns:d/1731236488974/Put/seqid=0 2024-11-10T11:01:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741842_1026 (size=5153) 2024-11-10T11:01:53,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741842_1026 (size=5153) 2024-11-10T11:01:53,900 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/ns/2ab74c65b8704872bed7f0dcfb567534 2024-11-10T11:01:53,919 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/table/f3fde7097c10487dab7c42c35b996397 is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1731236489402/Put/seqid=0 2024-11-10T11:01:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741843_1027 (size=5438) 2024-11-10T11:01:53,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741843_1027 (size=5438) 2024-11-10T11:01:53,924 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/table/f3fde7097c10487dab7c42c35b996397 2024-11-10T11:01:53,929 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/info/795aa29f80e84a6db743382b11674930 as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/info/795aa29f80e84a6db743382b11674930 2024-11-10T11:01:53,934 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/info/795aa29f80e84a6db743382b11674930, entries=10, sequenceid=11, filesize=7.0 K 2024-11-10T11:01:53,935 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/ns/2ab74c65b8704872bed7f0dcfb567534 as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/ns/2ab74c65b8704872bed7f0dcfb567534 2024-11-10T11:01:53,940 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/ns/2ab74c65b8704872bed7f0dcfb567534, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T11:01:53,941 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/.tmp/table/f3fde7097c10487dab7c42c35b996397 as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/table/f3fde7097c10487dab7c42c35b996397 2024-11-10T11:01:53,945 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/table/f3fde7097c10487dab7c42c35b996397, entries=2, sequenceid=11, filesize=5.3 K 2024-11-10T11:01:53,946 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 99ms, sequenceid=11, compaction requested=false 2024-11-10T11:01:53,947 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T11:01:53,947 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 33af8a6ffe292c16a5b1951425fba44a 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-10T11:01:53,947 ERROR [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,947 WARN [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8-prefix:eccb6f743e07,45733,1731236488002 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,948 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C45733%2C1731236488002:(num 1731236509818) roll requested 2024-11-10T11:01:53,948 INFO [regionserver/eccb6f743e07:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45733%2C1731236488002.1731236513948 2024-11-10T11:01:53,952 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 newFile=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236513948 2024-11-10T11:01:53,952 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,953 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,953 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,953 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,953 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:53,953 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236513948 2024-11-10T11:01:53,953 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,954 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-831653338-172.17.0.2-1731236487110:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:53,954 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:53,954 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 after 0ms 2024-11-10T11:01:53,954 DEBUG [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40609:40609),(127.0.0.1/127.0.0.1:33517:33517)] 2024-11-10T11:01:53,955 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.1731236509818 to hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs/eccb6f743e07%2C45733%2C1731236488002.1731236509818 2024-11-10T11:01:53,970 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/.tmp/info/12e3eaf530b34b89b22339541d89c923 is 1080, key is row1002/info:/1731236499134/Put/seqid=0 2024-11-10T11:01:53,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741845_1029 (size=9270) 2024-11-10T11:01:53,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741845_1029 (size=9270) 2024-11-10T11:01:53,975 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/.tmp/info/12e3eaf530b34b89b22339541d89c923 2024-11-10T11:01:53,982 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/.tmp/info/12e3eaf530b34b89b22339541d89c923 as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/info/12e3eaf530b34b89b22339541d89c923 2024-11-10T11:01:53,988 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/info/12e3eaf530b34b89b22339541d89c923, entries=4, sequenceid=8, filesize=9.1 K 2024-11-10T11:01:53,989 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 33af8a6ffe292c16a5b1951425fba44a in 42ms, sequenceid=8, compaction requested=false 2024-11-10T11:01:53,989 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 33af8a6ffe292c16a5b1951425fba44a: 2024-11-10T11:01:53,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:01:53,995 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:01:53,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:53,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:53,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:53,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:01:53,995 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:01:53,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1755805376, stopped=false 2024-11-10T11:01:53,996 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,44595,1731236487953 2024-11-10T11:01:53,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:53,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:01:53,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:53,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:53,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:01:53,998 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:01:53,998 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:53,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:53,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:53,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:01:53,998 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,45733,1731236488002' ***** 2024-11-10T11:01:53,998 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:01:53,999 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(3091): Received CLOSE for 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,45733,1731236488002 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:45733. 2024-11-10T11:01:53,999 DEBUG [RS:0;eccb6f743e07:45733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:01:53,999 DEBUG [RS:0;eccb6f743e07:45733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:53,999 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 33af8a6ffe292c16a5b1951425fba44a, disabling compactions & flushes 2024-11-10T11:01:53,999 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:01:53,999 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:54,000 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:01:54,000 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:54,000 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. after waiting 0 ms 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:54,000 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T11:01:54,000 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 33af8a6ffe292c16a5b1951425fba44a=TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a.} 2024-11-10T11:01:54,000 DEBUG [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 33af8a6ffe292c16a5b1951425fba44a 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:01:54,000 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:01:54,000 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:01:54,004 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/default/TestLogRolling-testLogRollOnPipelineRestart/33af8a6ffe292c16a5b1951425fba44a/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-10T11:01:54,004 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T11:01:54,005 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:54,005 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 33af8a6ffe292c16a5b1951425fba44a: Waiting for close lock at 1731236513999Running coprocessor pre-close hooks at 1731236513999Disabling compacts and flushes for region at 1731236513999Disabling writes for close at 1731236514000 (+1 ms)Writing region close event to WAL at 1731236514000Running coprocessor post-close hooks at 1731236514005 (+5 ms)Closed at 1731236514005 2024-11-10T11:01:54,005 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:01:54,005 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:01:54,005 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1731236489035.33af8a6ffe292c16a5b1951425fba44a. 2024-11-10T11:01:54,005 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236514000Running coprocessor pre-close hooks at 1731236514000Disabling compacts and flushes for region at 1731236514000Disabling writes for close at 1731236514000Writing region close event to WAL at 1731236514001 (+1 ms)Running coprocessor post-close hooks at 1731236514005 (+4 ms)Closed at 1731236514005 2024-11-10T11:01:54,005 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:01:54,200 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,45733,1731236488002; all regions closed. 2024-11-10T11:01:54,201 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:54,201 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:54,201 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:54,201 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:54,201 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:54,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741840_1023 (size=825) 2024-11-10T11:01:54,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741840_1023 (size=825) 2024-11-10T11:01:54,258 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:01:54,271 INFO [regionserver/eccb6f743e07:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T11:01:54,271 INFO [regionserver/eccb6f743e07:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T11:01:54,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:54,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:54,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:01:54,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:01:54,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-10T11:01:55,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:55,558 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:56,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:56,559 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:57,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:57,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:57,856 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta after 4001ms 2024-11-10T11:01:57,857 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/WALs/eccb6f743e07,45733,1731236488002/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta to hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs/eccb6f743e07%2C45733%2C1731236488002.meta.1731236488934.meta 2024-11-10T11:01:57,859 DEBUG [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs 2024-11-10T11:01:57,859 INFO [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C45733%2C1731236488002.meta:.meta(num 1731236513848) 2024-11-10T11:01:57,860 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,860 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,860 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,860 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,860 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741844_1028 (size=1162) 2024-11-10T11:01:57,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741844_1028 (size=1162) 2024-11-10T11:01:57,866 DEBUG [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs 2024-11-10T11:01:57,866 INFO [RS:0;eccb6f743e07:45733 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C45733%2C1731236488002:(num 1731236513948) 2024-11-10T11:01:57,866 DEBUG [RS:0;eccb6f743e07:45733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:01:57,866 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:01:57,866 INFO [RS:0;eccb6f743e07:45733 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:01:57,866 INFO [RS:0;eccb6f743e07:45733 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T11:01:57,866 INFO [RS:0;eccb6f743e07:45733 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:01:57,866 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:01:57,867 INFO [RS:0;eccb6f743e07:45733 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45733 2024-11-10T11:01:57,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,45733,1731236488002 2024-11-10T11:01:57,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:01:57,869 INFO [RS:0;eccb6f743e07:45733 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:01:57,870 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,45733,1731236488002] 2024-11-10T11:01:57,872 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,45733,1731236488002 already deleted, retry=false 2024-11-10T11:01:57,872 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,45733,1731236488002 expired; onlineServers=0 2024-11-10T11:01:57,872 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,44595,1731236487953' ***** 2024-11-10T11:01:57,872 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:01:57,872 INFO [M:0;eccb6f743e07:44595 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:01:57,872 INFO [M:0;eccb6f743e07:44595 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:01:57,872 DEBUG [M:0;eccb6f743e07:44595 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:01:57,872 DEBUG [M:0;eccb6f743e07:44595 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:01:57,872 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:01:57,872 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236488180 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236488180,5,FailOnTimeoutGroup] 2024-11-10T11:01:57,872 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236488180 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236488180,5,FailOnTimeoutGroup] 2024-11-10T11:01:57,873 INFO [M:0;eccb6f743e07:44595 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:01:57,873 INFO [M:0;eccb6f743e07:44595 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:01:57,873 DEBUG [M:0;eccb6f743e07:44595 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:01:57,873 INFO [M:0;eccb6f743e07:44595 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:01:57,873 INFO [M:0;eccb6f743e07:44595 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:01:57,873 INFO [M:0;eccb6f743e07:44595 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:01:57,873 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:01:57,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:01:57,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:01:57,876 DEBUG [M:0;eccb6f743e07:44595 {}] zookeeper.ZKUtil(347): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:01:57,876 WARN [M:0;eccb6f743e07:44595 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:01:57,876 INFO [M:0;eccb6f743e07:44595 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/.lastflushedseqids 2024-11-10T11:01:57,881 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T11:01:57,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741846_1030 (size=139) 2024-11-10T11:01:57,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741846_1030 (size=139) 2024-11-10T11:01:57,882 INFO [M:0;eccb6f743e07:44595 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:01:57,882 INFO [M:0;eccb6f743e07:44595 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:01:57,882 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:01:57,882 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:57,883 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:57,883 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:01:57,883 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:57,883 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB 2024-11-10T11:01:57,883 ERROR [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData-prefix:eccb6f743e07,44595,1731236487953 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:57,883 WARN [FSHLog-0-hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData-prefix:eccb6f743e07,44595,1731236487953 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:57,883 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog eccb6f743e07%2C44595%2C1731236487953:(num 1731236488097) roll requested 2024-11-10T11:01:57,884 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C44595%2C1731236487953.1731236517883 2024-11-10T11:01:57,888 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,888 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,888 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,888 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,888 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:57,888 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236517883 2024-11-10T11:01:57,889 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:57,889 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:40115,DS-dc24f1e2-4926-43c6-a645-b610a5df7533,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-10T11:01:57,889 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 2024-11-10T11:01:57,889 WARN [IPC Server handler 0 on default port 43083 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1014 2024-11-10T11:01:57,890 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 after 1ms 2024-11-10T11:01:57,893 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33517:33517),(127.0.0.1/127.0.0.1:40609:40609)] 2024-11-10T11:01:57,893 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 is not closed yet, will try archiving it next time 2024-11-10T11:01:57,908 DEBUG [M:0;eccb6f743e07:44595 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0dd66708641d494f8ec26fdeee69411e is 82, key is hbase:meta,,1/info:regioninfo/1731236488959/Put/seqid=0 2024-11-10T11:01:57,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741848_1033 (size=5672) 2024-11-10T11:01:57,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741848_1033 (size=5672) 2024-11-10T11:01:57,913 INFO [M:0;eccb6f743e07:44595 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0dd66708641d494f8ec26fdeee69411e 2024-11-10T11:01:57,932 DEBUG [M:0;eccb6f743e07:44595 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d36cfcaef1a249219f8de7abcf2daaa7 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731236489406/Put/seqid=0 2024-11-10T11:01:57,936 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T11:01:57,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741849_1034 (size=6118) 2024-11-10T11:01:57,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741849_1034 (size=6118) 2024-11-10T11:01:57,937 INFO [M:0;eccb6f743e07:44595 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d36cfcaef1a249219f8de7abcf2daaa7 2024-11-10T11:01:57,956 DEBUG [M:0;eccb6f743e07:44595 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/959a8798d503453db096512d154c6afb is 69, key is eccb6f743e07,45733,1731236488002/rs:state/1731236488237/Put/seqid=0 2024-11-10T11:01:57,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741850_1035 (size=5156) 2024-11-10T11:01:57,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741850_1035 (size=5156) 2024-11-10T11:01:57,962 INFO [M:0;eccb6f743e07:44595 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/959a8798d503453db096512d154c6afb 2024-11-10T11:01:57,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:57,971 INFO [RS:0;eccb6f743e07:45733 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:01:57,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45733-0x101018a4d920001, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:01:57,971 INFO [RS:0;eccb6f743e07:45733 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,45733,1731236488002; zookeeper connection closed. 2024-11-10T11:01:57,971 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d0b764c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d0b764c 2024-11-10T11:01:57,971 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T11:01:57,981 DEBUG [M:0;eccb6f743e07:44595 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7fe6cd5c6bbc4258bb17b305704bd449 is 52, key is load_balancer_on/state:d/1731236489030/Put/seqid=0 2024-11-10T11:01:57,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741851_1036 (size=5056) 2024-11-10T11:01:57,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741851_1036 (size=5056) 2024-11-10T11:01:57,986 INFO [M:0;eccb6f743e07:44595 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7fe6cd5c6bbc4258bb17b305704bd449 2024-11-10T11:01:57,992 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0dd66708641d494f8ec26fdeee69411e as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0dd66708641d494f8ec26fdeee69411e 2024-11-10T11:01:57,997 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0dd66708641d494f8ec26fdeee69411e, entries=8, sequenceid=56, filesize=5.5 K 2024-11-10T11:01:57,998 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d36cfcaef1a249219f8de7abcf2daaa7 as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d36cfcaef1a249219f8de7abcf2daaa7 2024-11-10T11:01:58,003 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d36cfcaef1a249219f8de7abcf2daaa7, entries=6, sequenceid=56, filesize=6.0 K 2024-11-10T11:01:58,003 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/959a8798d503453db096512d154c6afb as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/959a8798d503453db096512d154c6afb 2024-11-10T11:01:58,008 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/959a8798d503453db096512d154c6afb, entries=1, sequenceid=56, filesize=5.0 K 2024-11-10T11:01:58,009 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/7fe6cd5c6bbc4258bb17b305704bd449 as hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7fe6cd5c6bbc4258bb17b305704bd449 2024-11-10T11:01:58,014 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/7fe6cd5c6bbc4258bb17b305704bd449, entries=1, sequenceid=56, filesize=4.9 K 2024-11-10T11:01:58,015 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=56, compaction requested=false 2024-11-10T11:01:58,017 INFO [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:01:58,017 DEBUG [M:0;eccb6f743e07:44595 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236517882Disabling compacts and flushes for region at 1731236517882Disabling writes for close at 1731236517883 (+1 ms)Obtaining lock to block concurrent updates at 1731236517883Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236517883Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1731236517883Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236517893 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236517893Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236517907 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236517908 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236517918 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236517932 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236517932Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236517942 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236517956 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236517956Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236517967 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236517981 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236517981Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14230ff3: reopening flushed file at 1731236517991 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19f90233: reopening flushed file at 1731236517997 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a003dfc: reopening flushed file at 1731236518003 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4028d789: reopening flushed file at 1731236518008 (+5 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 132ms, sequenceid=56, compaction requested=false at 1731236518015 (+7 ms)Writing region close event to WAL at 1731236518017 (+2 ms)Closed at 1731236518017 2024-11-10T11:01:58,017 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:58,017 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:58,017 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:58,017 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:58,018 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:01:58,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39429 is added to blk_1073741847_1031 (size=757) 2024-11-10T11:01:58,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41935 is added to blk_1073741847_1031 (size=757) 2024-11-10T11:01:58,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:58,560 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:59,005 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,006 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,019 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,019 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,019 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,019 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,019 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,019 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,023 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,025 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,028 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,029 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,531 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:01:59,532 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,532 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,533 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,533 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,546 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,547 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,548 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,551 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,553 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:01:59,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:01:59,561 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:00,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:00,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:00,881 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1014: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-10T11:02:01,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:01,562 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:01,890 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 after 4001ms 2024-11-10T11:02:01,891 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/WALs/eccb6f743e07,44595,1731236487953/eccb6f743e07%2C44595%2C1731236487953.1731236488097 to hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/oldWALs/eccb6f743e07%2C44595%2C1731236487953.1731236488097 2024-11-10T11:02:01,894 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/MasterData/oldWALs/eccb6f743e07%2C44595%2C1731236487953.1731236488097 to hdfs://localhost:43083/user/jenkins/test-data/0325f7b0-eaed-5ab4-c93b-fd93490cbdd8/oldWALs/eccb6f743e07%2C44595%2C1731236487953.1731236488097$masterlocalwal$ 2024-11-10T11:02:01,894 INFO [M:0;eccb6f743e07:44595 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:02:01,894 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:02:01,894 INFO [M:0;eccb6f743e07:44595 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44595 2024-11-10T11:02:01,894 INFO [M:0;eccb6f743e07:44595 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:02:01,996 INFO [M:0;eccb6f743e07:44595 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:02:01,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:02:01,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44595-0x101018a4d920000, quorum=127.0.0.1:52878, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:02:01,999 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@21404da7{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:01,999 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e8914c8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:02:01,999 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:02:01,999 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6baabd83{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:02:01,999 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3146549d{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:02:02,001 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:02:02,001 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:02:02,001 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-831653338-172.17.0.2-1731236487110 (Datanode Uuid 29328916-b262-4879-ae98-dfb609c1d371) service to localhost/127.0.0.1:43083 2024-11-10T11:02:02,001 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:02:02,001 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data3/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:02,002 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data4/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:02,002 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:02:02,004 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1dcfabbb{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:02,004 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6614137b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:02:02,004 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:02:02,004 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62802d30{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:02:02,004 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@717499fb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:02:02,006 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:02:02,006 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:02:02,006 WARN [BP-831653338-172.17.0.2-1731236487110 heartbeating to localhost/127.0.0.1:43083 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-831653338-172.17.0.2-1731236487110 (Datanode Uuid 3a712815-02f0-4d5d-b159-846cbc068add) service to localhost/127.0.0.1:43083 2024-11-10T11:02:02,006 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:02:02,006 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data1/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:02,007 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/cluster_cb5a350b-7537-ed9a-7641-b5e472642490/data/data2/current/BP-831653338-172.17.0.2-1731236487110 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:02,007 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:02:02,012 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@65e5bfc{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:02:02,013 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1c23e5e9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:02:02,013 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:02:02,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6dbadaea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:02:02,013 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a3130af{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir/,STOPPED} 2024-11-10T11:02:02,019 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:02:02,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:02:02,045 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 153) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43083 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:43083 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43083 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43083 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43083 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:43083 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43083 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:43083 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=65 (was 104), ProcessCount=11 (was 11), AvailableMemoryMB=8689 (was 8865) 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=65, ProcessCount=11, AvailableMemoryMB=8689 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.log.dir so I do NOT create it in target/test-data/3783e692-4e1b-8b02-0344-24b013294769 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72982bdf-a481-50f3-a986-90f76e2c2d96/hadoop.tmp.dir so I do NOT create it in target/test-data/3783e692-4e1b-8b02-0344-24b013294769 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5, deleteOnExit=true 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T11:02:02,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/test.cache.data in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T11:02:02,054 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:02:02,054 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T11:02:02,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/nfs.dump.dir in system properties and HBase conf 2024-11-10T11:02:02,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/java.io.tmpdir in system properties and HBase conf 2024-11-10T11:02:02,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:02:02,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T11:02:02,055 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T11:02:02,068 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:02:02,135 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:02,139 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:02:02,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:02:02,140 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:02:02,140 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:02:02,141 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:02,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70e475c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:02:02,143 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290f188f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:02:02,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@62558ec9{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/java.io.tmpdir/jetty-localhost-36293-hadoop-hdfs-3_4_1-tests_jar-_-any-9110748746783807612/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:02:02,259 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@294894f5{HTTP/1.1, (http/1.1)}{localhost:36293} 2024-11-10T11:02:02,259 INFO [Time-limited test {}] server.Server(415): Started @182830ms 2024-11-10T11:02:02,272 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:02:02,326 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:02,330 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:02:02,331 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:02:02,331 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:02:02,331 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:02:02,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e6ee234{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:02:02,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b5b6aa1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:02:02,452 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@48bfafbe{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/java.io.tmpdir/jetty-localhost-34469-hadoop-hdfs-3_4_1-tests_jar-_-any-2064791073753780793/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:02,452 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b93cee9{HTTP/1.1, (http/1.1)}{localhost:34469} 2024-11-10T11:02:02,452 INFO [Time-limited test {}] server.Server(415): Started @183023ms 2024-11-10T11:02:02,453 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:02:02,484 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:02,487 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:02:02,488 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:02:02,488 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:02:02,488 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:02:02,488 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:02:02,489 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:02:02,549 WARN [Thread-1637 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data1/current/BP-1140510013-172.17.0.2-1731236522083/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:02,549 WARN [Thread-1638 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data2/current/BP-1140510013-172.17.0.2-1731236522083/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:02,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:02,563 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:02,573 WARN [Thread-1616 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:02:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb3e3d6b80932a21 with lease ID 0xff9b24a0edb8ed82: Processing first storage report for DS-1d82a8b9-41b7-47e5-ae76-fa77fa4c4e35 from datanode DatanodeRegistration(127.0.0.1:33135, datanodeUuid=98aff9be-00a8-436c-bc3a-dd079b0db114, infoPort=38561, infoSecurePort=0, ipcPort=44449, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083) 2024-11-10T11:02:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb3e3d6b80932a21 with lease ID 0xff9b24a0edb8ed82: from storage DS-1d82a8b9-41b7-47e5-ae76-fa77fa4c4e35 node DatanodeRegistration(127.0.0.1:33135, datanodeUuid=98aff9be-00a8-436c-bc3a-dd079b0db114, infoPort=38561, infoSecurePort=0, ipcPort=44449, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb3e3d6b80932a21 with lease ID 0xff9b24a0edb8ed82: Processing first storage report for DS-9163ae68-582a-4fda-bf69-2f9a9501945d from datanode DatanodeRegistration(127.0.0.1:33135, datanodeUuid=98aff9be-00a8-436c-bc3a-dd079b0db114, infoPort=38561, infoSecurePort=0, ipcPort=44449, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083) 2024-11-10T11:02:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb3e3d6b80932a21 with lease ID 0xff9b24a0edb8ed82: from storage DS-9163ae68-582a-4fda-bf69-2f9a9501945d node DatanodeRegistration(127.0.0.1:33135, datanodeUuid=98aff9be-00a8-436c-bc3a-dd079b0db114, infoPort=38561, infoSecurePort=0, ipcPort=44449, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:02,612 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@fc981fd{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/java.io.tmpdir/jetty-localhost-36327-hadoop-hdfs-3_4_1-tests_jar-_-any-16993212768540378416/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:02,612 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:36327} 2024-11-10T11:02:02,612 INFO [Time-limited test {}] server.Server(415): Started @183183ms 2024-11-10T11:02:02,613 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:02:02,717 WARN [Thread-1663 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data3/current/BP-1140510013-172.17.0.2-1731236522083/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:02,717 WARN [Thread-1664 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data4/current/BP-1140510013-172.17.0.2-1731236522083/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:02,734 WARN [Thread-1652 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:02:02,736 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdc1df4a10b901e5b with lease ID 0xff9b24a0edb8ed83: Processing first storage report for DS-168137fd-6fd1-4637-92d6-0a432df66bee from datanode DatanodeRegistration(127.0.0.1:35463, datanodeUuid=faf4becd-4b03-49fb-b9bf-9105f464c3de, infoPort=44363, infoSecurePort=0, ipcPort=38309, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083) 2024-11-10T11:02:02,736 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc1df4a10b901e5b with lease ID 0xff9b24a0edb8ed83: from storage DS-168137fd-6fd1-4637-92d6-0a432df66bee node DatanodeRegistration(127.0.0.1:35463, datanodeUuid=faf4becd-4b03-49fb-b9bf-9105f464c3de, infoPort=44363, infoSecurePort=0, ipcPort=38309, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:02,736 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xdc1df4a10b901e5b with lease ID 0xff9b24a0edb8ed83: Processing first storage report for DS-6baa8788-f416-4c83-969c-a9ebba955e7e from datanode DatanodeRegistration(127.0.0.1:35463, datanodeUuid=faf4becd-4b03-49fb-b9bf-9105f464c3de, infoPort=44363, infoSecurePort=0, ipcPort=38309, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083) 2024-11-10T11:02:02,736 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xdc1df4a10b901e5b with lease ID 0xff9b24a0edb8ed83: from storage DS-6baa8788-f416-4c83-969c-a9ebba955e7e node DatanodeRegistration(127.0.0.1:35463, datanodeUuid=faf4becd-4b03-49fb-b9bf-9105f464c3de, infoPort=44363, infoSecurePort=0, ipcPort=38309, storageInfo=lv=-57;cid=testClusterID;nsid=180097638;c=1731236522083), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:02,838 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769 2024-11-10T11:02:02,840 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/zookeeper_0, clientPort=50360, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T11:02:02,841 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50360 2024-11-10T11:02:02,842 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,843 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:02:02,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:02:02,852 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c with version=8 2024-11-10T11:02:02,853 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T11:02:02,855 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T11:02:02,855 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:02:02,856 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36107 2024-11-10T11:02:02,857 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36107 connecting to ZooKeeper ensemble=127.0.0.1:50360 2024-11-10T11:02:02,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361070x0, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:02:02,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36107-0x101018ad5e80000 connected 2024-11-10T11:02:02,880 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,882 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,883 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:02:02,884 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c, hbase.cluster.distributed=false 2024-11-10T11:02:02,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:02:02,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36107 2024-11-10T11:02:02,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36107 2024-11-10T11:02:02,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36107 2024-11-10T11:02:02,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36107 2024-11-10T11:02:02,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36107 2024-11-10T11:02:02,902 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:02:02,902 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:02:02,903 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46873 2024-11-10T11:02:02,904 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46873 connecting to ZooKeeper ensemble=127.0.0.1:50360 2024-11-10T11:02:02,905 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,907 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468730x0, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:02:02,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468730x0, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:02:02,911 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46873-0x101018ad5e80001 connected 2024-11-10T11:02:02,911 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:02:02,912 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:02:02,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T11:02:02,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:02:02,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46873 2024-11-10T11:02:02,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46873 2024-11-10T11:02:02,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46873 2024-11-10T11:02:02,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46873 2024-11-10T11:02:02,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46873 2024-11-10T11:02:02,926 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:36107 2024-11-10T11:02:02,927 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,36107,1731236522854 2024-11-10T11:02:02,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:02,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:02,929 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,36107,1731236522854 2024-11-10T11:02:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T11:02:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:02,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:02,933 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:02:02,933 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,36107,1731236522854 from backup master directory 2024-11-10T11:02:02,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,36107,1731236522854 2024-11-10T11:02:02,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:02,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:02,934 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:02:02,934 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,36107,1731236522854 2024-11-10T11:02:02,939 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/hbase.id] with ID: f6509072-f312-452a-8ed4-1e33ecd60f28 2024-11-10T11:02:02,939 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/.tmp/hbase.id 2024-11-10T11:02:02,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:02:02,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:02:02,945 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/.tmp/hbase.id]:[hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/hbase.id] 2024-11-10T11:02:02,956 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:02,956 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T11:02:02,957 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T11:02:02,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:02,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:02,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:02:02,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:02:02,966 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:02:02,967 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T11:02:02,967 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:02:02,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:02:02,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:02:02,976 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store 2024-11-10T11:02:02,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:02:02,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:02:02,985 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:02,985 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:02:02,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:02,985 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:02,985 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:02:02,985 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:02,985 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:02,986 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236522985Disabling compacts and flushes for region at 1731236522985Disabling writes for close at 1731236522985Writing region close event to WAL at 1731236522985Closed at 1731236522985 2024-11-10T11:02:02,986 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/.initializing 2024-11-10T11:02:02,986 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/WALs/eccb6f743e07,36107,1731236522854 2024-11-10T11:02:02,989 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36107%2C1731236522854, suffix=, logDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/WALs/eccb6f743e07,36107,1731236522854, archiveDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/oldWALs, maxLogs=10 2024-11-10T11:02:02,989 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36107%2C1731236522854.1731236522989 2024-11-10T11:02:02,995 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/WALs/eccb6f743e07,36107,1731236522854/eccb6f743e07%2C36107%2C1731236522854.1731236522989 2024-11-10T11:02:02,997 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38561:38561),(127.0.0.1/127.0.0.1:44363:44363)] 2024-11-10T11:02:02,998 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:02:02,999 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:02,999 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:02,999 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T11:02:03,002 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,002 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,003 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T11:02:03,003 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:03,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T11:02:03,005 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:03,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T11:02:03,007 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:03,007 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,008 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,008 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,009 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,009 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,010 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T11:02:03,011 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:03,013 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:02:03,014 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=752786, jitterRate=-0.04278428852558136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T11:02:03,014 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236522999Initializing all the Stores at 1731236523000 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523000Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236523000Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236523000Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236523000Cleaning up temporary data from old regions at 1731236523009 (+9 ms)Region opened successfully at 1731236523014 (+5 ms) 2024-11-10T11:02:03,014 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T11:02:03,017 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340993e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:02:03,018 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T11:02:03,018 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T11:02:03,018 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T11:02:03,018 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T11:02:03,019 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T11:02:03,019 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T11:02:03,019 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T11:02:03,021 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T11:02:03,022 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T11:02:03,023 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T11:02:03,023 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T11:02:03,024 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T11:02:03,028 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T11:02:03,028 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T11:02:03,029 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T11:02:03,030 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T11:02:03,031 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T11:02:03,032 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T11:02:03,034 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T11:02:03,035 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T11:02:03,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:02:03,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:02:03,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,038 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,36107,1731236522854, sessionid=0x101018ad5e80000, setting cluster-up flag (Was=false) 2024-11-10T11:02:03,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,047 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T11:02:03,047 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,36107,1731236522854 2024-11-10T11:02:03,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,056 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T11:02:03,057 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,36107,1731236522854 2024-11-10T11:02:03,058 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T11:02:03,060 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:03,060 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T11:02:03,060 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T11:02:03,061 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,36107,1731236522854 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:02:03,062 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236553063 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T11:02:03,063 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T11:02:03,063 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:03,064 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T11:02:03,064 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,064 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T11:02:03,064 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T11:02:03,064 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T11:02:03,064 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T11:02:03,064 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T11:02:03,065 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,065 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236523064,5,FailOnTimeoutGroup] 2024-11-10T11:02:03,065 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236523065,5,FailOnTimeoutGroup] 2024-11-10T11:02:03,065 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,065 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T11:02:03,065 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T11:02:03,065 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,065 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:02:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:02:03,071 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T11:02:03,072 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c 2024-11-10T11:02:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:02:03,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:02:03,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:03,079 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:02:03,080 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:02:03,080 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,080 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,081 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:02:03,082 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:02:03,082 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,082 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:02:03,083 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:02:03,083 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,084 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:02:03,085 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:02:03,085 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,085 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:02:03,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740 2024-11-10T11:02:03,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740 2024-11-10T11:02:03,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:02:03,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:02:03,088 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:02:03,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:02:03,090 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:02:03,091 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=763766, jitterRate=-0.028821706771850586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:02:03,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236523078Initializing all the Stores at 1731236523078Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523079 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523079Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236523079Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523079Cleaning up temporary data from old regions at 1731236523087 (+8 ms)Region opened successfully at 1731236523091 (+4 ms) 2024-11-10T11:02:03,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:02:03,091 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:02:03,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:02:03,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:02:03,091 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:02:03,092 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:02:03,092 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236523091Disabling compacts and flushes for region at 1731236523091Disabling writes for close at 1731236523091Writing region close event to WAL at 1731236523092 (+1 ms)Closed at 1731236523092 2024-11-10T11:02:03,093 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:03,093 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T11:02:03,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T11:02:03,094 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:02:03,096 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T11:02:03,116 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(746): ClusterId : f6509072-f312-452a-8ed4-1e33ecd60f28 2024-11-10T11:02:03,117 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:02:03,119 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:02:03,119 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:02:03,121 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:02:03,122 DEBUG [RS:0;eccb6f743e07:46873 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad521c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:02:03,133 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:46873 2024-11-10T11:02:03,134 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:02:03,134 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:02:03,134 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:02:03,134 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,36107,1731236522854 with port=46873, startcode=1731236522902 2024-11-10T11:02:03,135 DEBUG [RS:0;eccb6f743e07:46873 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:02:03,136 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46415, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:02:03,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,137 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36107 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,139 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c 2024-11-10T11:02:03,139 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:36001 2024-11-10T11:02:03,139 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:02:03,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:02:03,141 DEBUG [RS:0;eccb6f743e07:46873 {}] zookeeper.ZKUtil(111): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,141 WARN [RS:0;eccb6f743e07:46873 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:02:03,141 INFO [RS:0;eccb6f743e07:46873 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:02:03,141 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,142 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,46873,1731236522902] 2024-11-10T11:02:03,145 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:02:03,146 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:02:03,147 INFO [RS:0;eccb6f743e07:46873 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:02:03,147 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,147 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:02:03,148 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:02:03,148 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,148 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,149 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:03,149 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:02:03,149 DEBUG [RS:0;eccb6f743e07:46873 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:02:03,149 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,149 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,149 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,149 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,149 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,149 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,46873,1731236522902-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:02:03,164 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:02:03,164 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,46873,1731236522902-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,164 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,164 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.Replication(171): eccb6f743e07,46873,1731236522902 started 2024-11-10T11:02:03,178 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,178 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,46873,1731236522902, RpcServer on eccb6f743e07/172.17.0.2:46873, sessionid=0x101018ad5e80001 2024-11-10T11:02:03,178 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:02:03,178 DEBUG [RS:0;eccb6f743e07:46873 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,178 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,46873,1731236522902' 2024-11-10T11:02:03,178 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:02:03,179 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:02:03,179 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:02:03,179 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:02:03,179 DEBUG [RS:0;eccb6f743e07:46873 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,179 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,46873,1731236522902' 2024-11-10T11:02:03,179 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:02:03,180 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:02:03,180 DEBUG [RS:0;eccb6f743e07:46873 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:02:03,180 INFO [RS:0;eccb6f743e07:46873 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:02:03,180 INFO [RS:0;eccb6f743e07:46873 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:02:03,246 WARN [eccb6f743e07:36107 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T11:02:03,282 INFO [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C46873%2C1731236522902, suffix=, logDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902, archiveDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/oldWALs, maxLogs=32 2024-11-10T11:02:03,282 INFO [RS:0;eccb6f743e07:46873 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46873%2C1731236522902.1731236523282 2024-11-10T11:02:03,288 INFO [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236523282 2024-11-10T11:02:03,289 DEBUG [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44363:44363),(127.0.0.1/127.0.0.1:38561:38561)] 2024-11-10T11:02:03,496 DEBUG [eccb6f743e07:36107 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T11:02:03,497 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,498 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,46873,1731236522902, state=OPENING 2024-11-10T11:02:03,499 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T11:02:03,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:03,502 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:02:03,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:03,502 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:03,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,46873,1731236522902}] 2024-11-10T11:02:03,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:03,564 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:03,655 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T11:02:03,657 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60201, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T11:02:03,660 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T11:02:03,660 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:02:03,662 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C46873%2C1731236522902.meta, suffix=.meta, logDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902, archiveDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/oldWALs, maxLogs=32 2024-11-10T11:02:03,663 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46873%2C1731236522902.meta.1731236523662.meta 2024-11-10T11:02:03,669 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.meta.1731236523662.meta 2024-11-10T11:02:03,672 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44363:44363),(127.0.0.1/127.0.0.1:38561:38561)] 2024-11-10T11:02:03,673 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:02:03,673 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T11:02:03,673 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T11:02:03,673 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T11:02:03,673 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T11:02:03,673 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:03,674 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T11:02:03,674 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T11:02:03,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:02:03,676 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:02:03,676 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,677 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:02:03,677 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:02:03,677 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,678 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:02:03,678 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:02:03,678 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,679 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:02:03,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:02:03,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:03,680 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:02:03,681 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740 2024-11-10T11:02:03,681 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740 2024-11-10T11:02:03,682 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:02:03,682 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:02:03,683 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:02:03,684 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:02:03,684 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=695313, jitterRate=-0.11586436629295349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:02:03,685 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T11:02:03,685 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236523674Writing region info on filesystem at 1731236523674Initializing all the Stores at 1731236523674Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523674Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523675 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236523675Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236523675Cleaning up temporary data from old regions at 1731236523682 (+7 ms)Running coprocessor post-open hooks at 1731236523685 (+3 ms)Region opened successfully at 1731236523685 2024-11-10T11:02:03,686 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236523655 2024-11-10T11:02:03,689 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T11:02:03,689 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T11:02:03,689 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,690 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,46873,1731236522902, state=OPEN 2024-11-10T11:02:03,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:02:03,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:02:03,697 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,46873,1731236522902 2024-11-10T11:02:03,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:03,697 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:03,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T11:02:03,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,46873,1731236522902 in 195 msec 2024-11-10T11:02:03,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T11:02:03,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-11-10T11:02:03,703 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:03,703 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T11:02:03,704 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:02:03,704 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,46873,1731236522902, seqNum=-1] 2024-11-10T11:02:03,705 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:02:03,706 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33091, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:02:03,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 650 msec 2024-11-10T11:02:03,711 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236523711, completionTime=-1 2024-11-10T11:02:03,712 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T11:02:03,712 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T11:02:03,713 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T11:02:03,713 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236583713 2024-11-10T11:02:03,713 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236643713 2024-11-10T11:02:03,713 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec 2024-11-10T11:02:03,714 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36107,1731236522854-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,714 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36107,1731236522854-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,714 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36107,1731236522854-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,714 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:36107, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,714 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,714 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,715 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T11:02:03,717 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.782sec 2024-11-10T11:02:03,717 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T11:02:03,717 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T11:02:03,717 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T11:02:03,717 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T11:02:03,718 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T11:02:03,718 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36107,1731236522854-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:02:03,718 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36107,1731236522854-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T11:02:03,720 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T11:02:03,720 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T11:02:03,720 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36107,1731236522854-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:03,817 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2a30ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:02:03,817 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,36107,-1 for getting cluster id 2024-11-10T11:02:03,817 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T11:02:03,819 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f6509072-f312-452a-8ed4-1e33ecd60f28' 2024-11-10T11:02:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T11:02:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f6509072-f312-452a-8ed4-1e33ecd60f28" 2024-11-10T11:02:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76226915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:02:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,36107,-1] 2024-11-10T11:02:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T11:02:03,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:03,821 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T11:02:03,822 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30873421, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:02:03,823 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:02:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,46873,1731236522902, seqNum=-1] 2024-11-10T11:02:03,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:02:03,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:02:03,827 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,36107,1731236522854 2024-11-10T11:02:03,827 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:03,829 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T11:02:03,830 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T11:02:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is eccb6f743e07,36107,1731236522854 2024-11-10T11:02:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3b9f49a7 2024-11-10T11:02:03,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T11:02:03,832 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T11:02:03,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T11:02:03,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T11:02:03,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:02:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:03,835 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T11:02:03,835 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:03,835 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-10T11:02:03,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:02:03,836 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T11:02:03,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741835_1011 (size=405) 2024-11-10T11:02:03,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741835_1011 (size=405) 2024-11-10T11:02:03,845 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 914289af7cb94e1182b76a1d319425ad, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c 2024-11-10T11:02:03,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741836_1012 (size=88) 2024-11-10T11:02:03,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741836_1012 (size=88) 2024-11-10T11:02:03,851 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:03,851 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing 914289af7cb94e1182b76a1d319425ad, disabling compactions & flushes 2024-11-10T11:02:03,851 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:03,851 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:03,851 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. after waiting 0 ms 2024-11-10T11:02:03,851 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:03,851 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:03,851 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 914289af7cb94e1182b76a1d319425ad: Waiting for close lock at 1731236523851Disabling compacts and flushes for region at 1731236523851Disabling writes for close at 1731236523851Writing region close event to WAL at 1731236523851Closed at 1731236523851 2024-11-10T11:02:03,853 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T11:02:03,854 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1731236523853"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236523853"}]},"ts":"1731236523853"} 2024-11-10T11:02:03,856 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T11:02:03,858 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T11:02:03,858 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236523858"}]},"ts":"1731236523858"} 2024-11-10T11:02:03,861 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-10T11:02:03,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=914289af7cb94e1182b76a1d319425ad, ASSIGN}] 2024-11-10T11:02:03,862 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=914289af7cb94e1182b76a1d319425ad, ASSIGN 2024-11-10T11:02:03,863 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=914289af7cb94e1182b76a1d319425ad, ASSIGN; state=OFFLINE, location=eccb6f743e07,46873,1731236522902; forceNewPlan=false, retain=false 2024-11-10T11:02:04,014 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=914289af7cb94e1182b76a1d319425ad, regionState=OPENING, regionLocation=eccb6f743e07,46873,1731236522902 2024-11-10T11:02:04,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=914289af7cb94e1182b76a1d319425ad, ASSIGN because future has completed 2024-11-10T11:02:04,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914289af7cb94e1182b76a1d319425ad, server=eccb6f743e07,46873,1731236522902}] 2024-11-10T11:02:04,174 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:04,174 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 914289af7cb94e1182b76a1d319425ad, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:02:04,174 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,174 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:04,174 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,174 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,176 INFO [StoreOpener-914289af7cb94e1182b76a1d319425ad-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,177 INFO [StoreOpener-914289af7cb94e1182b76a1d319425ad-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 914289af7cb94e1182b76a1d319425ad columnFamilyName info 2024-11-10T11:02:04,177 DEBUG [StoreOpener-914289af7cb94e1182b76a1d319425ad-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:04,177 INFO [StoreOpener-914289af7cb94e1182b76a1d319425ad-1 {}] regionserver.HStore(327): Store=914289af7cb94e1182b76a1d319425ad/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:04,178 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,178 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,178 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,179 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,179 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,180 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,182 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:02:04,182 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 914289af7cb94e1182b76a1d319425ad; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=881967, jitterRate=0.12148015201091766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T11:02:04,183 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:04,184 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 914289af7cb94e1182b76a1d319425ad: Running coprocessor pre-open hook at 1731236524174Writing region info on filesystem at 1731236524174Initializing all the Stores at 1731236524175 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236524175Cleaning up temporary data from old regions at 1731236524179 (+4 ms)Running coprocessor post-open hooks at 1731236524183 (+4 ms)Region opened successfully at 1731236524183 2024-11-10T11:02:04,185 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad., pid=6, masterSystemTime=1731236524170 2024-11-10T11:02:04,187 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:04,187 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:04,188 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=914289af7cb94e1182b76a1d319425ad, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,46873,1731236522902 2024-11-10T11:02:04,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 914289af7cb94e1182b76a1d319425ad, server=eccb6f743e07,46873,1731236522902 because future has completed 2024-11-10T11:02:04,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T11:02:04,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 914289af7cb94e1182b76a1d319425ad, server=eccb6f743e07,46873,1731236522902 in 174 msec 2024-11-10T11:02:04,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T11:02:04,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=914289af7cb94e1182b76a1d319425ad, ASSIGN in 333 msec 2024-11-10T11:02:04,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T11:02:04,198 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236524198"}]},"ts":"1731236524198"} 2024-11-10T11:02:04,199 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-10T11:02:04,201 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T11:02:04,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 368 msec 2024-11-10T11:02:04,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:04,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:04,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:02:04,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T11:02:04,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:02:04,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T11:02:04,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:04,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T11:02:05,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:05,565 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:06,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:06,566 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:07,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:07,567 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:08,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:08,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:09,175 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:02:09,176 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,177 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,177 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,177 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,177 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,178 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,193 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,194 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,194 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,194 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,197 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,198 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,200 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:09,206 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T11:02:09,206 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-10T11:02:09,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:09,568 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:10,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:10,569 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:11,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:11,570 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:12,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:12,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:13,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:13,571 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:02:13,937 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T11:02:13,937 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-10T11:02:13,940 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:13,941 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:13,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad., hostname=eccb6f743e07,46873,1731236522902, seqNum=2] 2024-11-10T11:02:13,950 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:13,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:13,956 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T11:02:13,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T11:02:13,957 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T11:02:13,959 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T11:02:14,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T11:02:14,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:14,120 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 914289af7cb94e1182b76a1d319425ad 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T11:02:14,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/233cdcc34d3644cfb47cd5f0287cb367 is 1080, key is row0001/info:/1731236533944/Put/seqid=0 2024-11-10T11:02:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741837_1013 (size=6033) 2024-11-10T11:02:14,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741837_1013 (size=6033) 2024-11-10T11:02:14,141 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/233cdcc34d3644cfb47cd5f0287cb367 2024-11-10T11:02:14,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/233cdcc34d3644cfb47cd5f0287cb367 as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/233cdcc34d3644cfb47cd5f0287cb367 2024-11-10T11:02:14,153 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/233cdcc34d3644cfb47cd5f0287cb367, entries=1, sequenceid=5, filesize=5.9 K 2024-11-10T11:02:14,154 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 914289af7cb94e1182b76a1d319425ad in 35ms, sequenceid=5, compaction requested=false 2024-11-10T11:02:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 914289af7cb94e1182b76a1d319425ad: 2024-11-10T11:02:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:14,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T11:02:14,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T11:02:14,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T11:02:14,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 200 msec 2024-11-10T11:02:14,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 210 msec 2024-11-10T11:02:14,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:14,572 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:15,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:15,573 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:16,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:16,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:17,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:17,574 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:18,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:18,575 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:19,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:19,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:20,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:20,576 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:21,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:21,577 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:22,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:22,578 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:23,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:23,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:24,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T11:02:24,027 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T11:02:24,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:24,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T11:02:24,032 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T11:02:24,033 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T11:02:24,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T11:02:24,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T11:02:24,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:24,187 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 914289af7cb94e1182b76a1d319425ad 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T11:02:24,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/c93bb56049924384a7730ca46c52fecd is 1080, key is row0002/info:/1731236544028/Put/seqid=0 2024-11-10T11:02:24,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741838_1014 (size=6033) 2024-11-10T11:02:24,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741838_1014 (size=6033) 2024-11-10T11:02:24,197 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/c93bb56049924384a7730ca46c52fecd 2024-11-10T11:02:24,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/c93bb56049924384a7730ca46c52fecd as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/c93bb56049924384a7730ca46c52fecd 2024-11-10T11:02:24,209 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/c93bb56049924384a7730ca46c52fecd, entries=1, sequenceid=9, filesize=5.9 K 2024-11-10T11:02:24,210 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 914289af7cb94e1182b76a1d319425ad in 23ms, sequenceid=9, compaction requested=false 2024-11-10T11:02:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 914289af7cb94e1182b76a1d319425ad: 2024-11-10T11:02:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:24,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T11:02:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T11:02:24,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T11:02:24,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-10T11:02:24,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec 2024-11-10T11:02:24,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:24,579 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:25,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:25,580 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:26,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:26,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:27,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:27,581 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:28,582 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:28,582 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:29,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:29,583 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:29,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta after 68069ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:02:29,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 after 68083ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor203.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T11:02:30,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:30,584 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:31,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:31,585 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:32,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:32,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:32,838 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T11:02:33,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:33,586 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T11:02:34,097 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T11:02:34,101 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46873%2C1731236522902.1731236554100 2024-11-10T11:02:34,108 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:34,108 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:34,108 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:34,108 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:34,109 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:34,109 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236523282 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236554100 2024-11-10T11:02:34,112 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44363:44363),(127.0.0.1/127.0.0.1:38561:38561)] 2024-11-10T11:02:34,112 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236523282 is not closed yet, will try archiving it next time 2024-11-10T11:02:34,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741833_1009 (size=5546) 2024-11-10T11:02:34,113 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:34,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741833_1009 (size=5546) 2024-11-10T11:02:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:34,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T11:02:34,116 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T11:02:34,117 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T11:02:34,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T11:02:34,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T11:02:34,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:34,271 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 914289af7cb94e1182b76a1d319425ad 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T11:02:34,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/668415e971174fc9a42b57d90a37626b is 1080, key is row0003/info:/1731236554099/Put/seqid=0 2024-11-10T11:02:34,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741840_1016 (size=6033) 2024-11-10T11:02:34,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741840_1016 (size=6033) 2024-11-10T11:02:34,281 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/668415e971174fc9a42b57d90a37626b 2024-11-10T11:02:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/668415e971174fc9a42b57d90a37626b as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/668415e971174fc9a42b57d90a37626b 2024-11-10T11:02:34,293 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/668415e971174fc9a42b57d90a37626b, entries=1, sequenceid=13, filesize=5.9 K 2024-11-10T11:02:34,294 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 914289af7cb94e1182b76a1d319425ad in 23ms, sequenceid=13, compaction requested=true 2024-11-10T11:02:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 914289af7cb94e1182b76a1d319425ad: 2024-11-10T11:02:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:34,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T11:02:34,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T11:02:34,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T11:02:34,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-10T11:02:34,300 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec 2024-11-10T11:02:34,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:34,587 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:35,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:35,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:36,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:36,588 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:37,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:37,589 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:38,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:38,590 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:39,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:39,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:40,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:40,591 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:41,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:41,592 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:42,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:42,593 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:43,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:43,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T11:02:44,177 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T11:02:44,177 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:02:44,178 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:02:44,178 DEBUG [Time-limited test {}] regionserver.HStore(1541): 914289af7cb94e1182b76a1d319425ad/info is initiating minor compaction (all files) 2024-11-10T11:02:44,178 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:02:44,178 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:44,178 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 914289af7cb94e1182b76a1d319425ad/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:44,178 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/233cdcc34d3644cfb47cd5f0287cb367, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/c93bb56049924384a7730ca46c52fecd, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/668415e971174fc9a42b57d90a37626b] into tmpdir=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp, totalSize=17.7 K 2024-11-10T11:02:44,179 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 233cdcc34d3644cfb47cd5f0287cb367, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731236533944 2024-11-10T11:02:44,179 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c93bb56049924384a7730ca46c52fecd, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1731236544028 2024-11-10T11:02:44,179 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 668415e971174fc9a42b57d90a37626b, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731236554099 2024-11-10T11:02:44,189 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 914289af7cb94e1182b76a1d319425ad#info#compaction#44 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:02:44,190 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/5d00c82fe83b4ebf81a9de58ebe70c99 is 1080, key is row0001/info:/1731236533944/Put/seqid=0 2024-11-10T11:02:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741841_1017 (size=8296) 2024-11-10T11:02:44,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741841_1017 (size=8296) 2024-11-10T11:02:44,201 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/5d00c82fe83b4ebf81a9de58ebe70c99 as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/5d00c82fe83b4ebf81a9de58ebe70c99 2024-11-10T11:02:44,208 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 914289af7cb94e1182b76a1d319425ad/info of 914289af7cb94e1182b76a1d319425ad into 5d00c82fe83b4ebf81a9de58ebe70c99(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:02:44,208 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 914289af7cb94e1182b76a1d319425ad: 2024-11-10T11:02:44,210 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46873%2C1731236522902.1731236564210 2024-11-10T11:02:44,216 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:44,216 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:44,216 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:44,216 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:44,216 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:44,216 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236554100 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236564210 2024-11-10T11:02:44,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44363:44363),(127.0.0.1/127.0.0.1:38561:38561)] 2024-11-10T11:02:44,217 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236554100 is not closed yet, will try archiving it next time 2024-11-10T11:02:44,217 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236523282 to hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/oldWALs/eccb6f743e07%2C46873%2C1731236522902.1731236523282 2024-11-10T11:02:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741839_1015 (size=2520) 2024-11-10T11:02:44,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:44,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741839_1015 (size=2520) 2024-11-10T11:02:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:44,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T11:02:44,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-10T11:02:44,221 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T11:02:44,222 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T11:02:44,356 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T11:02:44,356 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T11:02:44,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46873 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T11:02:44,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:44,375 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 914289af7cb94e1182b76a1d319425ad 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T11:02:44,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/7211fe0ec3294fcb8af3b8b2da11af26 is 1080, key is row0000/info:/1731236564209/Put/seqid=0 2024-11-10T11:02:44,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741843_1019 (size=6033) 2024-11-10T11:02:44,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741843_1019 (size=6033) 2024-11-10T11:02:44,385 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/7211fe0ec3294fcb8af3b8b2da11af26 2024-11-10T11:02:44,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/7211fe0ec3294fcb8af3b8b2da11af26 as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/7211fe0ec3294fcb8af3b8b2da11af26 2024-11-10T11:02:44,394 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/7211fe0ec3294fcb8af3b8b2da11af26, entries=1, sequenceid=18, filesize=5.9 K 2024-11-10T11:02:44,396 INFO [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 914289af7cb94e1182b76a1d319425ad in 20ms, sequenceid=18, compaction requested=false 2024-11-10T11:02:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 914289af7cb94e1182b76a1d319425ad: 2024-11-10T11:02:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:44,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T11:02:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T11:02:44,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T11:02:44,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 175 msec 2024-11-10T11:02:44,402 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 182 msec 2024-11-10T11:02:44,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:44,594 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:44,619 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236554100 to hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/oldWALs/eccb6f743e07%2C46873%2C1731236522902.1731236554100 2024-11-10T11:02:45,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:45,595 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:46,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:46,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:47,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:47,596 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:48,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:48,597 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:49,174 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 914289af7cb94e1182b76a1d319425ad, had cached 0 bytes from a total of 14329 2024-11-10T11:02:49,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:49,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:50,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:50,598 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:51,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:51,599 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:52,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:52,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:53,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:53,600 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36107 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T11:02:54,307 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-10T11:02:54,310 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C46873%2C1731236522902.1731236574310 2024-11-10T11:02:54,315 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,315 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,315 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,315 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,315 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,315 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236564210 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236574310 2024-11-10T11:02:54,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:44363:44363),(127.0.0.1/127.0.0.1:38561:38561)] 2024-11-10T11:02:54,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/WALs/eccb6f743e07,46873,1731236522902/eccb6f743e07%2C46873%2C1731236522902.1731236564210 is not closed yet, will try archiving it next time 2024-11-10T11:02:54,316 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:02:54,316 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:02:54,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:02:54,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:54,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:54,316 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:02:54,317 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:02:54,317 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=273226305, stopped=false 2024-11-10T11:02:54,317 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,36107,1731236522854 2024-11-10T11:02:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741842_1018 (size=2026) 2024-11-10T11:02:54,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741842_1018 (size=2026) 2024-11-10T11:02:54,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:02:54,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:02:54,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:54,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:54,319 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:02:54,319 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:02:54,319 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:02:54,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:54,319 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,46873,1731236522902' ***** 2024-11-10T11:02:54,319 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:02:54,319 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:02:54,320 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(3091): Received CLOSE for 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,46873,1731236522902 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:02:54,320 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:46873. 2024-11-10T11:02:54,320 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 914289af7cb94e1182b76a1d319425ad, disabling compactions & flushes 2024-11-10T11:02:54,320 DEBUG [RS:0;eccb6f743e07:46873 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:02:54,320 DEBUG [RS:0;eccb6f743e07:46873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:54,320 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:54,320 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:02:54,320 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. after waiting 0 ms 2024-11-10T11:02:54,320 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:54,320 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:02:54,321 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 914289af7cb94e1182b76a1d319425ad 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-10T11:02:54,321 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T11:02:54,321 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 914289af7cb94e1182b76a1d319425ad=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.} 2024-11-10T11:02:54,321 DEBUG [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 914289af7cb94e1182b76a1d319425ad 2024-11-10T11:02:54,321 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:02:54,321 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:02:54,321 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:02:54,321 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:02:54,321 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:02:54,321 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-10T11:02:54,325 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/a3809b4c52754fa1b9ab663cd0366d5f is 1080, key is row0001/info:/1731236574308/Put/seqid=0 2024-11-10T11:02:54,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741845_1021 (size=6033) 2024-11-10T11:02:54,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741845_1021 (size=6033) 2024-11-10T11:02:54,336 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/a3809b4c52754fa1b9ab663cd0366d5f 2024-11-10T11:02:54,339 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/info/f77d0b18345e488a81fef730ac1cc58a is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad./info:regioninfo/1731236524188/Put/seqid=0 2024-11-10T11:02:54,342 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/.tmp/info/a3809b4c52754fa1b9ab663cd0366d5f as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/a3809b4c52754fa1b9ab663cd0366d5f 2024-11-10T11:02:54,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741846_1022 (size=7308) 2024-11-10T11:02:54,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741846_1022 (size=7308) 2024-11-10T11:02:54,345 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/info/f77d0b18345e488a81fef730ac1cc58a 2024-11-10T11:02:54,349 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/a3809b4c52754fa1b9ab663cd0366d5f, entries=1, sequenceid=22, filesize=5.9 K 2024-11-10T11:02:54,350 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 914289af7cb94e1182b76a1d319425ad in 30ms, sequenceid=22, compaction requested=true 2024-11-10T11:02:54,350 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/233cdcc34d3644cfb47cd5f0287cb367, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/c93bb56049924384a7730ca46c52fecd, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/668415e971174fc9a42b57d90a37626b] to archive 2024-11-10T11:02:54,351 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T11:02:54,352 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/233cdcc34d3644cfb47cd5f0287cb367 to hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/233cdcc34d3644cfb47cd5f0287cb367 2024-11-10T11:02:54,354 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/c93bb56049924384a7730ca46c52fecd to hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/c93bb56049924384a7730ca46c52fecd 2024-11-10T11:02:54,355 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/668415e971174fc9a42b57d90a37626b to hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/info/668415e971174fc9a42b57d90a37626b 2024-11-10T11:02:54,355 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=eccb6f743e07:36107 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-10T11:02:54,355 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [233cdcc34d3644cfb47cd5f0287cb367=6033, c93bb56049924384a7730ca46c52fecd=6033, 668415e971174fc9a42b57d90a37626b=6033] 2024-11-10T11:02:54,359 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/914289af7cb94e1182b76a1d319425ad/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-10T11:02:54,359 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:54,360 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 914289af7cb94e1182b76a1d319425ad: Waiting for close lock at 1731236574320Running coprocessor pre-close hooks at 1731236574320Disabling compacts and flushes for region at 1731236574320Disabling writes for close at 1731236574320Obtaining lock to block concurrent updates at 1731236574321 (+1 ms)Preparing flush snapshotting stores in 914289af7cb94e1182b76a1d319425ad at 1731236574321Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1731236574321Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. at 1731236574321Flushing 914289af7cb94e1182b76a1d319425ad/info: creating writer at 1731236574322 (+1 ms)Flushing 914289af7cb94e1182b76a1d319425ad/info: appending metadata at 1731236574324 (+2 ms)Flushing 914289af7cb94e1182b76a1d319425ad/info: closing flushed file at 1731236574324Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36115c97: reopening flushed file at 1731236574341 (+17 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 914289af7cb94e1182b76a1d319425ad in 30ms, sequenceid=22, compaction requested=true at 1731236574350 (+9 ms)Writing region close event to WAL at 1731236574356 (+6 ms)Running coprocessor post-close hooks at 1731236574359 (+3 ms)Closed at 1731236574359 2024-11-10T11:02:54,360 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1731236523832.914289af7cb94e1182b76a1d319425ad. 2024-11-10T11:02:54,365 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/ns/f9182a0995bd41e4b6cbabf127980fca is 43, key is default/ns:d/1731236523707/Put/seqid=0 2024-11-10T11:02:54,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741847_1023 (size=5153) 2024-11-10T11:02:54,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741847_1023 (size=5153) 2024-11-10T11:02:54,370 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/ns/f9182a0995bd41e4b6cbabf127980fca 2024-11-10T11:02:54,396 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/table/ea7956f1004140e2960094f65f092aec is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1731236524198/Put/seqid=0 2024-11-10T11:02:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741848_1024 (size=5508) 2024-11-10T11:02:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741848_1024 (size=5508) 2024-11-10T11:02:54,402 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/table/ea7956f1004140e2960094f65f092aec 2024-11-10T11:02:54,408 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/info/f77d0b18345e488a81fef730ac1cc58a as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/info/f77d0b18345e488a81fef730ac1cc58a 2024-11-10T11:02:54,413 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/info/f77d0b18345e488a81fef730ac1cc58a, entries=10, sequenceid=11, filesize=7.1 K 2024-11-10T11:02:54,414 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/ns/f9182a0995bd41e4b6cbabf127980fca as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/ns/f9182a0995bd41e4b6cbabf127980fca 2024-11-10T11:02:54,418 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/ns/f9182a0995bd41e4b6cbabf127980fca, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T11:02:54,419 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/.tmp/table/ea7956f1004140e2960094f65f092aec as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/table/ea7956f1004140e2960094f65f092aec 2024-11-10T11:02:54,424 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/table/ea7956f1004140e2960094f65f092aec, entries=2, sequenceid=11, filesize=5.4 K 2024-11-10T11:02:54,425 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false 2024-11-10T11:02:54,430 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T11:02:54,430 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:02:54,430 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:02:54,430 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236574321Running coprocessor pre-close hooks at 1731236574321Disabling compacts and flushes for region at 1731236574321Disabling writes for close at 1731236574321Obtaining lock to block concurrent updates at 1731236574321Preparing flush snapshotting stores in 1588230740 at 1731236574321Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1731236574321Flushing stores of hbase:meta,,1.1588230740 at 1731236574322 (+1 ms)Flushing 1588230740/info: creating writer at 1731236574322Flushing 1588230740/info: appending metadata at 1731236574338 (+16 ms)Flushing 1588230740/info: closing flushed file at 1731236574338Flushing 1588230740/ns: creating writer at 1731236574350 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731236574364 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731236574364Flushing 1588230740/table: creating writer at 1731236574375 (+11 ms)Flushing 1588230740/table: appending metadata at 1731236574396 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731236574396Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b9c02c8: reopening flushed file at 1731236574407 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1f077d: reopening flushed file at 1731236574413 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d054e56: reopening flushed file at 1731236574418 (+5 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false at 1731236574425 (+7 ms)Writing region close event to WAL at 1731236574426 (+1 ms)Running coprocessor post-close hooks at 1731236574430 (+4 ms)Closed at 1731236574430 2024-11-10T11:02:54,431 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:02:54,521 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,46873,1731236522902; all regions closed. 2024-11-10T11:02:54,521 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,522 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,522 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,522 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,522 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741834_1010 (size=3306) 2024-11-10T11:02:54,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741834_1010 (size=3306) 2024-11-10T11:02:54,526 DEBUG [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/oldWALs 2024-11-10T11:02:54,526 INFO [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C46873%2C1731236522902.meta:.meta(num 1731236523662) 2024-11-10T11:02:54,527 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,527 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,527 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,527 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,527 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741844_1020 (size=1252) 2024-11-10T11:02:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741844_1020 (size=1252) 2024-11-10T11:02:54,531 DEBUG [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/oldWALs 2024-11-10T11:02:54,532 INFO [RS:0;eccb6f743e07:46873 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C46873%2C1731236522902:(num 1731236574310) 2024-11-10T11:02:54,532 DEBUG [RS:0;eccb6f743e07:46873 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:54,532 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:02:54,532 INFO [RS:0;eccb6f743e07:46873 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:02:54,532 INFO [RS:0;eccb6f743e07:46873 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T11:02:54,532 INFO [RS:0;eccb6f743e07:46873 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:02:54,532 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:02:54,532 INFO [RS:0;eccb6f743e07:46873 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46873 2024-11-10T11:02:54,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,46873,1731236522902 2024-11-10T11:02:54,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:02:54,535 INFO [RS:0;eccb6f743e07:46873 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:02:54,536 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,46873,1731236522902] 2024-11-10T11:02:54,539 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,46873,1731236522902 already deleted, retry=false 2024-11-10T11:02:54,539 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,46873,1731236522902 expired; onlineServers=0 2024-11-10T11:02:54,539 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,36107,1731236522854' ***** 2024-11-10T11:02:54,539 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:02:54,539 INFO [M:0;eccb6f743e07:36107 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:02:54,539 INFO [M:0;eccb6f743e07:36107 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:02:54,539 DEBUG [M:0;eccb6f743e07:36107 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:02:54,539 DEBUG [M:0;eccb6f743e07:36107 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:02:54,539 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:02:54,539 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236523064 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236523064,5,FailOnTimeoutGroup] 2024-11-10T11:02:54,539 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236523065 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236523065,5,FailOnTimeoutGroup] 2024-11-10T11:02:54,539 INFO [M:0;eccb6f743e07:36107 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:02:54,539 INFO [M:0;eccb6f743e07:36107 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:02:54,540 DEBUG [M:0;eccb6f743e07:36107 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:02:54,540 INFO [M:0;eccb6f743e07:36107 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:02:54,540 INFO [M:0;eccb6f743e07:36107 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:02:54,540 INFO [M:0;eccb6f743e07:36107 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:02:54,540 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:02:54,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:02:54,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:54,541 DEBUG [M:0;eccb6f743e07:36107 {}] zookeeper.ZKUtil(347): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:02:54,541 WARN [M:0;eccb6f743e07:36107 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:02:54,541 INFO [M:0;eccb6f743e07:36107 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/.lastflushedseqids 2024-11-10T11:02:54,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741849_1025 (size=130) 2024-11-10T11:02:54,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741849_1025 (size=130) 2024-11-10T11:02:54,547 INFO [M:0;eccb6f743e07:36107 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:02:54,547 INFO [M:0;eccb6f743e07:36107 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:02:54,547 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:02:54,547 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:54,547 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:54,547 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:02:54,547 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:54,548 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.58 KB heapSize=54.99 KB 2024-11-10T11:02:54,563 DEBUG [M:0;eccb6f743e07:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a6382414b0f04f33956226939286404d is 82, key is hbase:meta,,1/info:regioninfo/1731236523689/Put/seqid=0 2024-11-10T11:02:54,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741850_1026 (size=5672) 2024-11-10T11:02:54,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741850_1026 (size=5672) 2024-11-10T11:02:54,568 INFO [M:0;eccb6f743e07:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a6382414b0f04f33956226939286404d 2024-11-10T11:02:54,587 DEBUG [M:0;eccb6f743e07:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68d15a3cba884a0fb22324136112e326 is 798, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731236524202/Put/seqid=0 2024-11-10T11:02:54,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741851_1027 (size=7822) 2024-11-10T11:02:54,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741851_1027 (size=7822) 2024-11-10T11:02:54,593 INFO [M:0;eccb6f743e07:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.98 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68d15a3cba884a0fb22324136112e326 2024-11-10T11:02:54,597 INFO [M:0;eccb6f743e07:36107 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68d15a3cba884a0fb22324136112e326 2024-11-10T11:02:54,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:54,601 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:54,611 DEBUG [M:0;eccb6f743e07:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7fd30a3d75cc45348e5d94659984a2ae is 69, key is eccb6f743e07,46873,1731236522902/rs:state/1731236523137/Put/seqid=0 2024-11-10T11:02:54,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741852_1028 (size=5156) 2024-11-10T11:02:54,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741852_1028 (size=5156) 2024-11-10T11:02:54,616 INFO [M:0;eccb6f743e07:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7fd30a3d75cc45348e5d94659984a2ae 2024-11-10T11:02:54,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:02:54,636 INFO [RS:0;eccb6f743e07:46873 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:02:54,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46873-0x101018ad5e80001, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:02:54,636 INFO [RS:0;eccb6f743e07:46873 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,46873,1731236522902; zookeeper connection closed. 2024-11-10T11:02:54,637 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1e863c00 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1e863c00 2024-11-10T11:02:54,637 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T11:02:54,642 DEBUG [M:0;eccb6f743e07:36107 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd95272d65644f8189e23c28a936f8f2 is 52, key is load_balancer_on/state:d/1731236523828/Put/seqid=0 2024-11-10T11:02:54,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741853_1029 (size=5056) 2024-11-10T11:02:54,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741853_1029 (size=5056) 2024-11-10T11:02:54,648 INFO [M:0;eccb6f743e07:36107 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd95272d65644f8189e23c28a936f8f2 2024-11-10T11:02:54,654 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a6382414b0f04f33956226939286404d as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a6382414b0f04f33956226939286404d 2024-11-10T11:02:54,659 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a6382414b0f04f33956226939286404d, entries=8, sequenceid=121, filesize=5.5 K 2024-11-10T11:02:54,659 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/68d15a3cba884a0fb22324136112e326 as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/68d15a3cba884a0fb22324136112e326 2024-11-10T11:02:54,664 INFO [M:0;eccb6f743e07:36107 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 68d15a3cba884a0fb22324136112e326 2024-11-10T11:02:54,664 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/68d15a3cba884a0fb22324136112e326, entries=14, sequenceid=121, filesize=7.6 K 2024-11-10T11:02:54,665 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7fd30a3d75cc45348e5d94659984a2ae as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7fd30a3d75cc45348e5d94659984a2ae 2024-11-10T11:02:54,669 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7fd30a3d75cc45348e5d94659984a2ae, entries=1, sequenceid=121, filesize=5.0 K 2024-11-10T11:02:54,669 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd95272d65644f8189e23c28a936f8f2 as hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bd95272d65644f8189e23c28a936f8f2 2024-11-10T11:02:54,674 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:36001/user/jenkins/test-data/f0535343-f697-1897-599d-b4b19428fe4c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bd95272d65644f8189e23c28a936f8f2, entries=1, sequenceid=121, filesize=4.9 K 2024-11-10T11:02:54,675 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.58 KB/44629, heapSize ~54.93 KB/56248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=121, compaction requested=false 2024-11-10T11:02:54,676 INFO [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:54,676 DEBUG [M:0;eccb6f743e07:36107 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236574547Disabling compacts and flushes for region at 1731236574547Disabling writes for close at 1731236574547Obtaining lock to block concurrent updates at 1731236574548 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236574548Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44629, getHeapSize=56248, getOffHeapSize=0, getCellsCount=140 at 1731236574548Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236574549 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236574549Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236574563 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236574563Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236574573 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236574587 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236574587Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236574597 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236574611 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236574611Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236574621 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236574642 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236574642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2233f333: reopening flushed file at 1731236574653 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36679ad2: reopening flushed file at 1731236574659 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14d546bf: reopening flushed file at 1731236574664 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ae60547: reopening flushed file at 1731236574669 (+5 ms)Finished flush of dataSize ~43.58 KB/44629, heapSize ~54.93 KB/56248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 128ms, sequenceid=121, compaction requested=false at 1731236574675 (+6 ms)Writing region close event to WAL at 1731236574676 (+1 ms)Closed at 1731236574676 2024-11-10T11:02:54,676 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,677 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,677 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,677 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,677 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:02:54,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35463 is added to blk_1073741830_1006 (size=53026) 2024-11-10T11:02:54,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33135 is added to blk_1073741830_1006 (size=53026) 2024-11-10T11:02:54,680 INFO [M:0;eccb6f743e07:36107 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:02:54,680 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:02:54,680 INFO [M:0;eccb6f743e07:36107 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36107 2024-11-10T11:02:54,680 INFO [M:0;eccb6f743e07:36107 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:02:54,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:02:54,782 INFO [M:0;eccb6f743e07:36107 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:02:54,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36107-0x101018ad5e80000, quorum=127.0.0.1:50360, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:02:54,784 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@fc981fd{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:54,784 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1367dc96{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:02:54,785 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:02:54,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@190ad9e7{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:02:54,785 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@20f59884{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir/,STOPPED} 2024-11-10T11:02:54,786 WARN [BP-1140510013-172.17.0.2-1731236522083 heartbeating to localhost/127.0.0.1:36001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:02:54,786 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:02:54,786 WARN [BP-1140510013-172.17.0.2-1731236522083 heartbeating to localhost/127.0.0.1:36001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1140510013-172.17.0.2-1731236522083 (Datanode Uuid faf4becd-4b03-49fb-b9bf-9105f464c3de) service to localhost/127.0.0.1:36001 2024-11-10T11:02:54,786 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:02:54,787 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data3/current/BP-1140510013-172.17.0.2-1731236522083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:54,787 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data4/current/BP-1140510013-172.17.0.2-1731236522083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:54,787 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:02:54,789 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@48bfafbe{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:54,789 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b93cee9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:02:54,790 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:02:54,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b5b6aa1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:02:54,790 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e6ee234{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir/,STOPPED} 2024-11-10T11:02:54,791 WARN [BP-1140510013-172.17.0.2-1731236522083 heartbeating to localhost/127.0.0.1:36001 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:02:54,791 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:02:54,791 WARN [BP-1140510013-172.17.0.2-1731236522083 heartbeating to localhost/127.0.0.1:36001 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1140510013-172.17.0.2-1731236522083 (Datanode Uuid 98aff9be-00a8-436c-bc3a-dd079b0db114) service to localhost/127.0.0.1:36001 2024-11-10T11:02:54,791 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:02:54,792 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data1/current/BP-1140510013-172.17.0.2-1731236522083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:54,792 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/cluster_9efa4b36-710b-9acd-d511-c50c6ab405c5/data/data2/current/BP-1140510013-172.17.0.2-1731236522083 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:02:54,792 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:02:54,798 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@62558ec9{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:02:54,798 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@294894f5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:02:54,798 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:02:54,798 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290f188f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:02:54,799 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70e475c9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir/,STOPPED} 2024-11-10T11:02:54,805 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:02:54,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:02:54,830 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=205 (was 179) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36001 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36001 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36001 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:36001 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:36001 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36001 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/eccb6f743e07:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:36001 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36001 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=39 (was 65), ProcessCount=11 (was 11), AvailableMemoryMB=8650 (was 8689) 2024-11-10T11:02:54,837 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=205, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=39, ProcessCount=11, AvailableMemoryMB=8649 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.log.dir so I do NOT create it in target/test-data/e7769794-76aa-2346-72cc-a678ab94564c 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3783e692-4e1b-8b02-0344-24b013294769/hadoop.tmp.dir so I do NOT create it in target/test-data/e7769794-76aa-2346-72cc-a678ab94564c 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac, deleteOnExit=true 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/test.cache.data in system properties and HBase conf 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir in system properties and HBase conf 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T11:02:54,838 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:02:54,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/nfs.dump.dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/java.io.tmpdir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T11:02:54,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T11:02:54,852 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:02:54,915 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:54,918 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:02:54,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:02:54,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:02:54,919 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:02:54,920 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:54,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3d320e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:02:54,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7e466bdb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:02:54,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:02:54,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:02:54,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T11:02:54,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-10T11:02:55,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2ffc76d1{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/java.io.tmpdir/jetty-localhost-39005-hadoop-hdfs-3_4_1-tests_jar-_-any-6576557828260298567/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:02:55,036 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@640c9e09{HTTP/1.1, (http/1.1)}{localhost:39005} 2024-11-10T11:02:55,036 INFO [Time-limited test {}] server.Server(415): Started @235607ms 2024-11-10T11:02:55,049 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:02:55,142 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:55,147 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:02:55,149 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:02:55,149 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:02:55,149 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:02:55,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a83b9f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:02:55,151 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3c10c155{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:02:55,152 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:02:55,277 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27e7702e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/java.io.tmpdir/jetty-localhost-38845-hadoop-hdfs-3_4_1-tests_jar-_-any-4227041401675360989/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:55,278 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@33382c80{HTTP/1.1, (http/1.1)}{localhost:38845} 2024-11-10T11:02:55,278 INFO [Time-limited test {}] server.Server(415): Started @235849ms 2024-11-10T11:02:55,279 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:02:55,308 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:02:55,311 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:02:55,311 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:02:55,311 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:02:55,311 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T11:02:55,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:02:55,312 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:02:55,385 WARN [Thread-1952 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data1/current/BP-1385195385-172.17.0.2-1731236574859/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:55,385 WARN [Thread-1953 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data2/current/BP-1385195385-172.17.0.2-1731236574859/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:55,402 WARN [Thread-1931 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:02:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b84187246834b with lease ID 0x778d767328189ea3: Processing first storage report for DS-79c141de-6375-492f-b0a7-93fc2d55f815 from datanode DatanodeRegistration(127.0.0.1:42175, datanodeUuid=886933b3-9da5-4304-9f40-94ffebbc8dfc, infoPort=43017, infoSecurePort=0, ipcPort=46289, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859) 2024-11-10T11:02:55,404 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b84187246834b with lease ID 0x778d767328189ea3: from storage DS-79c141de-6375-492f-b0a7-93fc2d55f815 node DatanodeRegistration(127.0.0.1:42175, datanodeUuid=886933b3-9da5-4304-9f40-94ffebbc8dfc, infoPort=43017, infoSecurePort=0, ipcPort=46289, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:55,405 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b84187246834b with lease ID 0x778d767328189ea3: Processing first storage report for DS-ab2b9b18-c536-4381-8c2f-da4e696f2b8c from datanode DatanodeRegistration(127.0.0.1:42175, datanodeUuid=886933b3-9da5-4304-9f40-94ffebbc8dfc, infoPort=43017, infoSecurePort=0, ipcPort=46289, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859) 2024-11-10T11:02:55,405 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b84187246834b with lease ID 0x778d767328189ea3: from storage DS-ab2b9b18-c536-4381-8c2f-da4e696f2b8c node DatanodeRegistration(127.0.0.1:42175, datanodeUuid=886933b3-9da5-4304-9f40-94ffebbc8dfc, infoPort=43017, infoSecurePort=0, ipcPort=46289, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-10T11:02:55,430 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@76385d85{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/java.io.tmpdir/jetty-localhost-42587-hadoop-hdfs-3_4_1-tests_jar-_-any-6315275095027189208/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:02:55,430 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:42587} 2024-11-10T11:02:55,430 INFO [Time-limited test {}] server.Server(415): Started @236001ms 2024-11-10T11:02:55,432 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:02:55,532 WARN [Thread-1979 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data4/current/BP-1385195385-172.17.0.2-1731236574859/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:55,532 WARN [Thread-1978 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data3/current/BP-1385195385-172.17.0.2-1731236574859/current, will proceed with Du for space computation calculation, 2024-11-10T11:02:55,549 WARN [Thread-1967 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:02:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b805a6bab04388 with lease ID 0x778d767328189ea4: Processing first storage report for DS-1e770c00-a8a9-48d3-958c-30a0e5ebb116 from datanode DatanodeRegistration(127.0.0.1:43697, datanodeUuid=8f52354b-388b-417a-b058-9ce882a6e6d6, infoPort=39541, infoSecurePort=0, ipcPort=38183, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859) 2024-11-10T11:02:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b805a6bab04388 with lease ID 0x778d767328189ea4: from storage DS-1e770c00-a8a9-48d3-958c-30a0e5ebb116 node DatanodeRegistration(127.0.0.1:43697, datanodeUuid=8f52354b-388b-417a-b058-9ce882a6e6d6, infoPort=39541, infoSecurePort=0, ipcPort=38183, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2b805a6bab04388 with lease ID 0x778d767328189ea4: Processing first storage report for DS-18eb357c-79bf-450a-9ccc-6c04472c7082 from datanode DatanodeRegistration(127.0.0.1:43697, datanodeUuid=8f52354b-388b-417a-b058-9ce882a6e6d6, infoPort=39541, infoSecurePort=0, ipcPort=38183, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859) 2024-11-10T11:02:55,552 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2b805a6bab04388 with lease ID 0x778d767328189ea4: from storage DS-18eb357c-79bf-450a-9ccc-6c04472c7082 node DatanodeRegistration(127.0.0.1:43697, datanodeUuid=8f52354b-388b-417a-b058-9ce882a6e6d6, infoPort=39541, infoSecurePort=0, ipcPort=38183, storageInfo=lv=-57;cid=testClusterID;nsid=769770776;c=1731236574859), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:02:55,553 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c 2024-11-10T11:02:55,556 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/zookeeper_0, clientPort=59422, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T11:02:55,557 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59422 2024-11-10T11:02:55,557 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,559 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:02:55,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:02:55,568 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0 with version=8 2024-11-10T11:02:55,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T11:02:55,570 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:02:55,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:55,570 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:55,571 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:02:55,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:55,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:02:55,571 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T11:02:55,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:02:55,572 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42865 2024-11-10T11:02:55,574 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42865 connecting to ZooKeeper ensemble=127.0.0.1:59422 2024-11-10T11:02:55,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428650x0, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:02:55,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42865-0x101018ba3d30000 connected 2024-11-10T11:02:55,597 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,598 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:02:55,600 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0, hbase.cluster.distributed=false 2024-11-10T11:02:55,601 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:02:55,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42865 2024-11-10T11:02:55,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:55,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:55,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42865 2024-11-10T11:02:55,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42865 2024-11-10T11:02:55,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42865 2024-11-10T11:02:55,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42865 2024-11-10T11:02:55,622 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:02:55,622 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:02:55,623 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36119 2024-11-10T11:02:55,624 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36119 connecting to ZooKeeper ensemble=127.0.0.1:59422 2024-11-10T11:02:55,625 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361190x0, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:02:55,631 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361190x0, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:02:55,632 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36119-0x101018ba3d30001 connected 2024-11-10T11:02:55,632 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:02:55,632 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:02:55,633 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T11:02:55,634 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:02:55,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36119 2024-11-10T11:02:55,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36119 2024-11-10T11:02:55,634 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36119 2024-11-10T11:02:55,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36119 2024-11-10T11:02:55,635 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36119 2024-11-10T11:02:55,647 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:42865 2024-11-10T11:02:55,647 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:55,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:55,649 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T11:02:55,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,652 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:02:55,652 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,42865,1731236575570 from backup master directory 2024-11-10T11:02:55,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:55,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:02:55,653 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:02:55,653 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,657 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/hbase.id] with ID: 8e3f3f8e-e011-4a04-9019-da2f0addb012 2024-11-10T11:02:55,657 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/.tmp/hbase.id 2024-11-10T11:02:55,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:02:55,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:02:55,663 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/.tmp/hbase.id]:[hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/hbase.id] 2024-11-10T11:02:55,673 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:55,673 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T11:02:55,674 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T11:02:55,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:02:55,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:02:55,683 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:02:55,684 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T11:02:55,684 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:02:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:02:55,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:02:55,691 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store 2024-11-10T11:02:55,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:02:55,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:02:55,697 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:55,697 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:02:55,697 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:55,697 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:55,697 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:02:55,697 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:55,697 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:02:55,697 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236575697Disabling compacts and flushes for region at 1731236575697Disabling writes for close at 1731236575697Writing region close event to WAL at 1731236575697Closed at 1731236575697 2024-11-10T11:02:55,698 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/.initializing 2024-11-10T11:02:55,698 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/WALs/eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,700 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C42865%2C1731236575570, suffix=, logDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/WALs/eccb6f743e07,42865,1731236575570, archiveDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/oldWALs, maxLogs=10 2024-11-10T11:02:55,700 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C42865%2C1731236575570.1731236575700 2024-11-10T11:02:55,704 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/WALs/eccb6f743e07,42865,1731236575570/eccb6f743e07%2C42865%2C1731236575570.1731236575700 2024-11-10T11:02:55,705 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39541:39541),(127.0.0.1/127.0.0.1:43017:43017)] 2024-11-10T11:02:55,706 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:02:55,706 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:55,706 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,706 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,707 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,708 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T11:02:55,708 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:55,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T11:02:55,710 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:55,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T11:02:55,711 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:55,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T11:02:55,712 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:55,713 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,713 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,714 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,715 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,715 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,715 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T11:02:55,716 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:02:55,718 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:02:55,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=836799, jitterRate=0.06404578685760498}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T11:02:55,719 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236575706Initializing all the Stores at 1731236575707 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236575707Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236575707Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236575707Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236575707Cleaning up temporary data from old regions at 1731236575715 (+8 ms)Region opened successfully at 1731236575719 (+4 ms) 2024-11-10T11:02:55,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T11:02:55,723 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@477e4a0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:02:55,723 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T11:02:55,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T11:02:55,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T11:02:55,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T11:02:55,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T11:02:55,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T11:02:55,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T11:02:55,726 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T11:02:55,727 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T11:02:55,729 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T11:02:55,729 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T11:02:55,729 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T11:02:55,731 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T11:02:55,731 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T11:02:55,731 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T11:02:55,733 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T11:02:55,733 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T11:02:55,736 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T11:02:55,737 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T11:02:55,738 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T11:02:55,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:02:55,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:02:55,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,741 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,42865,1731236575570, sessionid=0x101018ba3d30000, setting cluster-up flag (Was=false) 2024-11-10T11:02:55,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,749 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T11:02:55,749 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:55,759 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T11:02:55,760 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,42865,1731236575570 2024-11-10T11:02:55,761 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T11:02:55,763 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:55,763 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T11:02:55,763 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T11:02:55,763 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,42865,1731236575570 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:02:55,764 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,766 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:55,766 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T11:02:55,766 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236605766 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T11:02:55,767 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,767 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T11:02:55,767 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T11:02:55,768 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T11:02:55,768 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T11:02:55,768 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T11:02:55,768 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T11:02:55,768 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236575768,5,FailOnTimeoutGroup] 2024-11-10T11:02:55,772 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236575768,5,FailOnTimeoutGroup] 2024-11-10T11:02:55,772 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,772 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T11:02:55,772 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,772 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:02:55,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:02:55,778 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T11:02:55,778 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0 2024-11-10T11:02:55,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:02:55,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:02:55,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:55,787 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:02:55,788 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:02:55,789 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:55,789 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:02:55,790 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:02:55,790 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:55,790 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:02:55,791 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:02:55,791 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:55,792 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:02:55,793 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:02:55,793 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:55,793 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:55,793 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:02:55,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740 2024-11-10T11:02:55,794 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740 2024-11-10T11:02:55,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:02:55,795 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:02:55,795 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:02:55,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:02:55,798 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:02:55,798 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=819526, jitterRate=0.04208153486251831}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:02:55,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236575786Initializing all the Stores at 1731236575787 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236575787Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236575787Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236575787Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236575787Cleaning up temporary data from old regions at 1731236575795 (+8 ms)Region opened successfully at 1731236575799 (+4 ms) 2024-11-10T11:02:55,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:02:55,799 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:02:55,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:02:55,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:02:55,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:02:55,799 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:02:55,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236575799Disabling compacts and flushes for region at 1731236575799Disabling writes for close at 1731236575799Writing region close event to WAL at 1731236575799Closed at 1731236575799 2024-11-10T11:02:55,801 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:55,801 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T11:02:55,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T11:02:55,802 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:02:55,803 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T11:02:55,837 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(746): ClusterId : 8e3f3f8e-e011-4a04-9019-da2f0addb012 2024-11-10T11:02:55,837 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:02:55,839 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:02:55,839 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:02:55,840 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:02:55,841 DEBUG [RS:0;eccb6f743e07:36119 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc3aa75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:02:55,853 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:36119 2024-11-10T11:02:55,853 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:02:55,853 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:02:55,853 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:02:55,854 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,42865,1731236575570 with port=36119, startcode=1731236575621 2024-11-10T11:02:55,854 DEBUG [RS:0;eccb6f743e07:36119 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:02:55,856 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36843, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:02:55,856 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,36119,1731236575621 2024-11-10T11:02:55,856 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,36119,1731236575621 2024-11-10T11:02:55,858 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0 2024-11-10T11:02:55,858 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:41611 2024-11-10T11:02:55,858 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:02:55,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:02:55,860 DEBUG [RS:0;eccb6f743e07:36119 {}] zookeeper.ZKUtil(111): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,36119,1731236575621 2024-11-10T11:02:55,860 WARN [RS:0;eccb6f743e07:36119 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:02:55,860 INFO [RS:0;eccb6f743e07:36119 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:02:55,860 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621 2024-11-10T11:02:55,861 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,36119,1731236575621] 2024-11-10T11:02:55,863 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:02:55,865 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:02:55,865 INFO [RS:0;eccb6f743e07:36119 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:02:55,865 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,866 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:02:55,866 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:02:55,866 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:02:55,867 DEBUG [RS:0;eccb6f743e07:36119 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:02:55,867 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,867 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,867 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,867 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,867 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,867 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36119,1731236575621-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:02:55,882 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:02:55,882 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,36119,1731236575621-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,882 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,882 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.Replication(171): eccb6f743e07,36119,1731236575621 started 2024-11-10T11:02:55,896 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:55,896 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,36119,1731236575621, RpcServer on eccb6f743e07/172.17.0.2:36119, sessionid=0x101018ba3d30001 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,36119,1731236575621 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,36119,1731236575621' 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,36119,1731236575621 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,36119,1731236575621' 2024-11-10T11:02:55,897 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:02:55,898 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:02:55,898 DEBUG [RS:0;eccb6f743e07:36119 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:02:55,898 INFO [RS:0;eccb6f743e07:36119 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:02:55,898 INFO [RS:0;eccb6f743e07:36119 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:02:55,953 WARN [eccb6f743e07:42865 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T11:02:56,000 INFO [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36119%2C1731236575621, suffix=, logDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621, archiveDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/oldWALs, maxLogs=32 2024-11-10T11:02:56,000 INFO [RS:0;eccb6f743e07:36119 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36119%2C1731236575621.1731236576000 2024-11-10T11:02:56,006 INFO [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236576000 2024-11-10T11:02:56,007 DEBUG [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43017:43017),(127.0.0.1/127.0.0.1:39541:39541)] 2024-11-10T11:02:56,203 DEBUG [eccb6f743e07:42865 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T11:02:56,204 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:02:56,205 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,36119,1731236575621, state=OPENING 2024-11-10T11:02:56,208 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T11:02:56,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:56,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:02:56,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:56,210 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:56,210 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:02:56,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,36119,1731236575621}] 2024-11-10T11:02:56,363 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T11:02:56,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54073, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T11:02:56,368 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T11:02:56,368 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:02:56,369 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C36119%2C1731236575621.meta, suffix=.meta, logDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621, archiveDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/oldWALs, maxLogs=32 2024-11-10T11:02:56,370 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36119%2C1731236575621.meta.1731236576370.meta 2024-11-10T11:02:56,374 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.meta.1731236576370.meta 2024-11-10T11:02:56,375 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43017:43017),(127.0.0.1/127.0.0.1:39541:39541)] 2024-11-10T11:02:56,376 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:02:56,376 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T11:02:56,376 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T11:02:56,376 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T11:02:56,376 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T11:02:56,376 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:56,377 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T11:02:56,377 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T11:02:56,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:02:56,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:02:56,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:56,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:56,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:02:56,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:02:56,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:56,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:56,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:02:56,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:02:56,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:56,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:56,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:02:56,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:02:56,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:56,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:02:56,382 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:02:56,382 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740 2024-11-10T11:02:56,383 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740 2024-11-10T11:02:56,384 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:02:56,384 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:02:56,385 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:02:56,386 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:02:56,387 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=821315, jitterRate=0.04435661435127258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:02:56,387 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T11:02:56,387 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236576377Writing region info on filesystem at 1731236576377Initializing all the Stores at 1731236576377Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236576377Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236576377Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236576378 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236576378Cleaning up temporary data from old regions at 1731236576384 (+6 ms)Running coprocessor post-open hooks at 1731236576387 (+3 ms)Region opened successfully at 1731236576387 2024-11-10T11:02:56,388 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236576363 2024-11-10T11:02:56,391 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T11:02:56,391 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T11:02:56,392 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:02:56,393 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,36119,1731236575621, state=OPEN 2024-11-10T11:02:56,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:02:56,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:02:56,398 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,36119,1731236575621 2024-11-10T11:02:56,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:56,398 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:02:56,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T11:02:56,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,36119,1731236575621 in 188 msec 2024-11-10T11:02:56,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T11:02:56,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 599 msec 2024-11-10T11:02:56,403 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:02:56,403 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T11:02:56,404 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:02:56,404 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,36119,1731236575621, seqNum=-1] 2024-11-10T11:02:56,405 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:02:56,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57493, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:02:56,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 648 msec 2024-11-10T11:02:56,411 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236576411, completionTime=-1 2024-11-10T11:02:56,411 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T11:02:56,411 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T11:02:56,416 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T11:02:56,416 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236636416 2024-11-10T11:02:56,416 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236696416 2024-11-10T11:02:56,416 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 5 msec 2024-11-10T11:02:56,417 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,42865,1731236575570-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,417 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,42865,1731236575570-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,417 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,42865,1731236575570-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,417 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:42865, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,417 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,417 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,419 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.768sec 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,42865,1731236575570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:02:56,421 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,42865,1731236575570-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T11:02:56,423 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T11:02:56,423 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T11:02:56,423 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,42865,1731236575570-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:02:56,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c8059, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:02:56,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,42865,-1 for getting cluster id 2024-11-10T11:02:56,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T11:02:56,438 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '8e3f3f8e-e011-4a04-9019-da2f0addb012' 2024-11-10T11:02:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T11:02:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "8e3f3f8e-e011-4a04-9019-da2f0addb012" 2024-11-10T11:02:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@99463c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:02:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,42865,-1] 2024-11-10T11:02:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T11:02:56,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:02:56,441 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T11:02:56,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e87d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:02:56,442 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:02:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,36119,1731236575621, seqNum=-1] 2024-11-10T11:02:56,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:02:56,444 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:02:56,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,42865,1731236575570 2024-11-10T11:02:56,446 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:02:56,448 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T11:02:56,448 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T11:02:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is eccb6f743e07,42865,1731236575570 2024-11-10T11:02:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@b2ab9d7 2024-11-10T11:02:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T11:02:56,450 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T11:02:56,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-10T11:02:56,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-10T11:02:56,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:02:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-10T11:02:56,454 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T11:02:56,454 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:56,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-10T11:02:56,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:02:56,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T11:02:56,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741835_1011 (size=381) 2024-11-10T11:02:56,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741835_1011 (size=381) 2024-11-10T11:02:56,463 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1e67176538e0c6b983734c0b73e922ed, NAME => 'TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0 2024-11-10T11:02:56,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741836_1012 (size=64) 2024-11-10T11:02:56,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741836_1012 (size=64) 2024-11-10T11:02:56,470 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:56,470 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 1e67176538e0c6b983734c0b73e922ed, disabling compactions & flushes 2024-11-10T11:02:56,470 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,470 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,471 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. after waiting 0 ms 2024-11-10T11:02:56,471 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,471 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,471 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1e67176538e0c6b983734c0b73e922ed: Waiting for close lock at 1731236576470Disabling compacts and flushes for region at 1731236576470Disabling writes for close at 1731236576471 (+1 ms)Writing region close event to WAL at 1731236576471Closed at 1731236576471 2024-11-10T11:02:56,472 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T11:02:56,472 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731236576472"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236576472"}]},"ts":"1731236576472"} 2024-11-10T11:02:56,474 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T11:02:56,475 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T11:02:56,476 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236576475"}]},"ts":"1731236576475"} 2024-11-10T11:02:56,477 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-10T11:02:56,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, ASSIGN}] 2024-11-10T11:02:56,479 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, ASSIGN 2024-11-10T11:02:56,479 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, ASSIGN; state=OFFLINE, location=eccb6f743e07,36119,1731236575621; forceNewPlan=false, retain=false 2024-11-10T11:02:56,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:56,602 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:56,630 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1e67176538e0c6b983734c0b73e922ed, regionState=OPENING, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:02:56,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, ASSIGN because future has completed 2024-11-10T11:02:56,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e67176538e0c6b983734c0b73e922ed, server=eccb6f743e07,36119,1731236575621}] 2024-11-10T11:02:56,790 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,790 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1e67176538e0c6b983734c0b73e922ed, NAME => 'TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:02:56,790 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,790 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:02:56,790 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,790 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,792 INFO [StoreOpener-1e67176538e0c6b983734c0b73e922ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,793 INFO [StoreOpener-1e67176538e0c6b983734c0b73e922ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e67176538e0c6b983734c0b73e922ed columnFamilyName info 2024-11-10T11:02:56,793 DEBUG [StoreOpener-1e67176538e0c6b983734c0b73e922ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:02:56,794 INFO [StoreOpener-1e67176538e0c6b983734c0b73e922ed-1 {}] regionserver.HStore(327): Store=1e67176538e0c6b983734c0b73e922ed/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:02:56,794 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,794 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,794 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,795 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,795 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,796 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,798 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:02:56,798 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1e67176538e0c6b983734c0b73e922ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=812557, jitterRate=0.033220693469047546}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T11:02:56,798 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:02:56,799 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1e67176538e0c6b983734c0b73e922ed: Running coprocessor pre-open hook at 1731236576790Writing region info on filesystem at 1731236576790Initializing all the Stores at 1731236576791 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236576791Cleaning up temporary data from old regions at 1731236576795 (+4 ms)Running coprocessor post-open hooks at 1731236576798 (+3 ms)Region opened successfully at 1731236576799 (+1 ms) 2024-11-10T11:02:56,800 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., pid=6, masterSystemTime=1731236576786 2024-11-10T11:02:56,802 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,802 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:02:56,803 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1e67176538e0c6b983734c0b73e922ed, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:02:56,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1e67176538e0c6b983734c0b73e922ed, server=eccb6f743e07,36119,1731236575621 because future has completed 2024-11-10T11:02:56,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T11:02:56,808 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1e67176538e0c6b983734c0b73e922ed, server=eccb6f743e07,36119,1731236575621 in 173 msec 2024-11-10T11:02:56,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T11:02:56,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, ASSIGN in 331 msec 2024-11-10T11:02:56,811 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T11:02:56,812 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731236576811"}]},"ts":"1731236576811"} 2024-11-10T11:02:56,814 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-10T11:02:56,815 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T11:02:56,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 364 msec 2024-11-10T11:02:57,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:57,603 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:58,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:58,604 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:59,360 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,360 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,360 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,360 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,361 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,361 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,361 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,361 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,376 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,376 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,376 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,376 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,376 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,377 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,379 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,379 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,379 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,381 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:59,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:02:59,884 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:02:59,885 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,885 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,886 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,886 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,886 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,886 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,886 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,887 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,903 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,903 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,903 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,904 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,904 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,904 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,907 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,907 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,907 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:02:59,910 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:00,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:00,605 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:01,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:01,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:01,863 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T11:03:01,864 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-10T11:03:02,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:02,606 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:03,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:03,607 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:04,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:04,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:04,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:03:04,968 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T11:03:04,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:03:04,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T11:03:04,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T11:03:04,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T11:03:04,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-10T11:03:04,969 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-10T11:03:05,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:05,608 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:06,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42865 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T11:03:06,517 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-10T11:03:06,517 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-10T11:03:06,520 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-10T11:03:06,520 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:06,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., hostname=eccb6f743e07,36119,1731236575621, seqNum=2] 2024-11-10T11:03:06,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:06,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:03:06,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/906aa1596cc944f5b0be4f829d21d352 is 1080, key is row0001/info:/1731236586523/Put/seqid=0 2024-11-10T11:03:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741837_1013 (size=12509) 2024-11-10T11:03:06,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741837_1013 (size=12509) 2024-11-10T11:03:06,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/906aa1596cc944f5b0be4f829d21d352 2024-11-10T11:03:06,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/906aa1596cc944f5b0be4f829d21d352 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/906aa1596cc944f5b0be4f829d21d352 2024-11-10T11:03:06,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/906aa1596cc944f5b0be4f829d21d352, entries=7, sequenceid=11, filesize=12.2 K 2024-11-10T11:03:06,569 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=22.07 KB/22596 for 1e67176538e0c6b983734c0b73e922ed in 36ms, sequenceid=11, compaction requested=false 2024-11-10T11:03:06,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:06,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=24.17 KB heapSize=26.13 KB 2024-11-10T11:03:06,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/ad86411152754dc9817ce8cbe442f1e0 is 1080, key is row0008/info:/1731236586534/Put/seqid=0 2024-11-10T11:03:06,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741838_1014 (size=29761) 2024-11-10T11:03:06,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741838_1014 (size=29761) 2024-11-10T11:03:06,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.17 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/ad86411152754dc9817ce8cbe442f1e0 2024-11-10T11:03:06,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/ad86411152754dc9817ce8cbe442f1e0 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0 2024-11-10T11:03:06,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0, entries=23, sequenceid=37, filesize=29.1 K 2024-11-10T11:03:06,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.17 KB/24748, heapSize ~26.11 KB/26736, currentSize=2.10 KB/2152 for 1e67176538e0c6b983734c0b73e922ed in 19ms, sequenceid=37, compaction requested=false 2024-11-10T11:03:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=41.3 K, sizeToCheck=16.0 K 2024-11-10T11:03:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:06,590 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0 because midkey is the same as first or last row 2024-11-10T11:03:06,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:06,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:07,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:07,609 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:08,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:08,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:03:08,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/33215cc55cd04898b171cd9c1a52f15a is 1080, key is row0031/info:/1731236586571/Put/seqid=0 2024-11-10T11:03:08,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741839_1015 (size=12509) 2024-11-10T11:03:08,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741839_1015 (size=12509) 2024-11-10T11:03:08,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/33215cc55cd04898b171cd9c1a52f15a 2024-11-10T11:03:08,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/33215cc55cd04898b171cd9c1a52f15a as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/33215cc55cd04898b171cd9c1a52f15a 2024-11-10T11:03:08,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/33215cc55cd04898b171cd9c1a52f15a, entries=7, sequenceid=47, filesize=12.2 K 2024-11-10T11:03:08,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 1e67176538e0c6b983734c0b73e922ed in 24ms, sequenceid=47, compaction requested=true 2024-11-10T11:03:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:08,608 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=53.5 K, sizeToCheck=16.0 K 2024-11-10T11:03:08,609 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:08,609 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0 because midkey is the same as first or last row 2024-11-10T11:03:08,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e67176538e0c6b983734c0b73e922ed:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:08,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:08,609 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:08,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:08,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T11:03:08,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:08,610 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:08,611 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54779 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:08,611 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 1e67176538e0c6b983734c0b73e922ed/info is initiating minor compaction (all files) 2024-11-10T11:03:08,611 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e67176538e0c6b983734c0b73e922ed/info in TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:08,611 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/906aa1596cc944f5b0be4f829d21d352, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/33215cc55cd04898b171cd9c1a52f15a] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp, totalSize=53.5 K 2024-11-10T11:03:08,612 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 906aa1596cc944f5b0be4f829d21d352, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731236586523 2024-11-10T11:03:08,612 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad86411152754dc9817ce8cbe442f1e0, keycount=23, bloomtype=ROW, size=29.1 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731236586534 2024-11-10T11:03:08,612 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33215cc55cd04898b171cd9c1a52f15a, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731236586571 2024-11-10T11:03:08,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/b636c4e51d464912bbda151b0ad8ff82 is 1080, key is row0038/info:/1731236588585/Put/seqid=0 2024-11-10T11:03:08,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741840_1016 (size=21141) 2024-11-10T11:03:08,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741840_1016 (size=21141) 2024-11-10T11:03:08,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/b636c4e51d464912bbda151b0ad8ff82 2024-11-10T11:03:08,628 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e67176538e0c6b983734c0b73e922ed#info#compaction#58 average throughput is 12.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:08,628 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/8d77e0caad064ed7a1ae3116ab09dbb1 is 1080, key is row0001/info:/1731236586523/Put/seqid=0 2024-11-10T11:03:08,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/b636c4e51d464912bbda151b0ad8ff82 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/b636c4e51d464912bbda151b0ad8ff82 2024-11-10T11:03:08,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741841_1017 (size=44978) 2024-11-10T11:03:08,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741841_1017 (size=44978) 2024-11-10T11:03:08,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/b636c4e51d464912bbda151b0ad8ff82, entries=15, sequenceid=65, filesize=20.6 K 2024-11-10T11:03:08,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=12.61 KB/12912 for 1e67176538e0c6b983734c0b73e922ed in 29ms, sequenceid=65, compaction requested=false 2024-11-10T11:03:08,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:08,638 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=74.1 K, sizeToCheck=16.0 K 2024-11-10T11:03:08,638 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:08,638 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0 because midkey is the same as first or last row 2024-11-10T11:03:08,641 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/8d77e0caad064ed7a1ae3116ab09dbb1 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 2024-11-10T11:03:08,648 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e67176538e0c6b983734c0b73e922ed/info of 1e67176538e0c6b983734c0b73e922ed into 8d77e0caad064ed7a1ae3116ab09dbb1(size=43.9 K), total size for store is 64.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:08,648 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., storeName=1e67176538e0c6b983734c0b73e922ed/info, priority=13, startTime=1731236588609; duration=0sec 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 because midkey is the same as first or last row 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 because midkey is the same as first or last row 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=64.6 K, sizeToCheck=16.0 K 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 because midkey is the same as first or last row 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:08,648 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e67176538e0c6b983734c0b73e922ed:info 2024-11-10T11:03:09,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:09,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:10,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:10,611 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-10T11:03:10,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/7b6f2d60894f46f6835e76ab5629e730 is 1080, key is row0053/info:/1731236588610/Put/seqid=0 2024-11-10T11:03:10,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741842_1018 (size=18987) 2024-11-10T11:03:10,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741842_1018 (size=18987) 2024-11-10T11:03:10,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/7b6f2d60894f46f6835e76ab5629e730 2024-11-10T11:03:10,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/7b6f2d60894f46f6835e76ab5629e730 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/7b6f2d60894f46f6835e76ab5629e730 2024-11-10T11:03:10,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/7b6f2d60894f46f6835e76ab5629e730, entries=13, sequenceid=82, filesize=18.5 K 2024-11-10T11:03:10,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=13.66 KB/13988 for 1e67176538e0c6b983734c0b73e922ed in 22ms, sequenceid=82, compaction requested=true 2024-11-10T11:03:10,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:10,659 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=83.1 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,659 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,659 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 because midkey is the same as first or last row 2024-11-10T11:03:10,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1e67176538e0c6b983734c0b73e922ed:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:10,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:10,659 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T11:03:10,660 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:10,660 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 1e67176538e0c6b983734c0b73e922ed/info is initiating minor compaction (all files) 2024-11-10T11:03:10,660 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1e67176538e0c6b983734c0b73e922ed/info in TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:10,660 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/b636c4e51d464912bbda151b0ad8ff82, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/7b6f2d60894f46f6835e76ab5629e730] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp, totalSize=83.1 K 2024-11-10T11:03:10,661 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d77e0caad064ed7a1ae3116ab09dbb1, keycount=37, bloomtype=ROW, size=43.9 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731236586523 2024-11-10T11:03:10,661 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting b636c4e51d464912bbda151b0ad8ff82, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1731236588585 2024-11-10T11:03:10,662 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6f2d60894f46f6835e76ab5629e730, keycount=13, bloomtype=ROW, size=18.5 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731236588610 2024-11-10T11:03:10,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/3f29ecfe653449d9a0acdb54d47e45f7 is 1080, key is row0066/info:/1731236590637/Put/seqid=0 2024-11-10T11:03:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741843_1019 (size=20064) 2024-11-10T11:03:10,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741843_1019 (size=20064) 2024-11-10T11:03:10,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/3f29ecfe653449d9a0acdb54d47e45f7 2024-11-10T11:03:10,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/3f29ecfe653449d9a0acdb54d47e45f7 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/3f29ecfe653449d9a0acdb54d47e45f7 2024-11-10T11:03:10,676 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1e67176538e0c6b983734c0b73e922ed#info#compaction#61 average throughput is 22.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:10,676 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/30aab023d5244b8aae8eec9ddb98b730 is 1080, key is row0001/info:/1731236586523/Put/seqid=0 2024-11-10T11:03:10,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/3f29ecfe653449d9a0acdb54d47e45f7, entries=14, sequenceid=99, filesize=19.6 K 2024-11-10T11:03:10,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 1e67176538e0c6b983734c0b73e922ed in 24ms, sequenceid=99, compaction requested=false 2024-11-10T11:03:10,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:10,683 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=102.7 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,683 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,684 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 because midkey is the same as first or last row 2024-11-10T11:03:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T11:03:10,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741844_1020 (size=75378) 2024-11-10T11:03:10,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741844_1020 (size=75378) 2024-11-10T11:03:10,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/f11f2fd444a64c27aa5386c6e4de6e3b is 1080, key is row0080/info:/1731236590660/Put/seqid=0 2024-11-10T11:03:10,693 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/30aab023d5244b8aae8eec9ddb98b730 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730 2024-11-10T11:03:10,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741845_1021 (size=20064) 2024-11-10T11:03:10,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741845_1021 (size=20064) 2024-11-10T11:03:10,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/f11f2fd444a64c27aa5386c6e4de6e3b 2024-11-10T11:03:10,700 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1e67176538e0c6b983734c0b73e922ed/info of 1e67176538e0c6b983734c0b73e922ed into 30aab023d5244b8aae8eec9ddb98b730(size=73.6 K), total size for store is 93.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:10,700 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:10,700 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., storeName=1e67176538e0c6b983734c0b73e922ed/info, priority=13, startTime=1731236590659; duration=0sec 2024-11-10T11:03:10,701 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,701 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,701 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,701 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,701 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=93.2 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,701 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,702 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:10,702 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:10,702 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1e67176538e0c6b983734c0b73e922ed:info 2024-11-10T11:03:10,703 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] assignment.AssignmentManager(1355): Split request from eccb6f743e07,36119,1731236575621, parent={ENCODED => 1e67176538e0c6b983734c0b73e922ed, NAME => 'TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-10T11:03:10,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/f11f2fd444a64c27aa5386c6e4de6e3b as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/f11f2fd444a64c27aa5386c6e4de6e3b 2024-11-10T11:03:10,709 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:10,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/f11f2fd444a64c27aa5386c6e4de6e3b, entries=14, sequenceid=116, filesize=19.6 K 2024-11-10T11:03:10,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=3.15 KB/3228 for 1e67176538e0c6b983734c0b73e922ed in 25ms, sequenceid=116, compaction requested=true 2024-11-10T11:03:10,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1e67176538e0c6b983734c0b73e922ed: 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=112.8 K, sizeToCheck=16.0 K 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-10T11:03:10,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=1 2024-11-10T11:03:10,713 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=6dc8a2fd92092c8fc02930629a7fa4ce, daughterB=4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:10,714 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=6dc8a2fd92092c8fc02930629a7fa4ce, daughterB=4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:10,714 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=6dc8a2fd92092c8fc02930629a7fa4ce, daughterB=4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:10,714 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=6dc8a2fd92092c8fc02930629a7fa4ce, daughterB=4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:10,715 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] assignment.AssignmentManager(1355): Split request from eccb6f743e07,36119,1731236575621, parent={ENCODED => 1e67176538e0c6b983734c0b73e922ed, NAME => 'TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-10T11:03:10,716 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=SPLITTING, location=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:10,717 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42865 {}] procedure2.ProcedureExecutor(1139): Stored pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee 2024-11-10T11:03:10,717 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(842): Waiting on xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee held by pid=7 2024-11-10T11:03:10,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, UNASSIGN}] 2024-11-10T11:03:10,726 DEBUG [PEWorker-3 {}] assignment.SplitTableRegionProcedure(162): LOCK_EVENT_WAIT SchemaLocking[serverLocks={},namespaceLocks={hbase=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},tableLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},regionLocks={},peerLocks={},metaLocks={hbase:meta=LockAndQueue[exclusiveLock=false,sharedLockCount=0,waitingProcCount=0]},globalLocks={}] 2024-11-10T11:03:10,726 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, UNASSIGN 2024-11-10T11:03:10,726 DEBUG [PEWorker-3 {}] procedure2.ProcedureExecutor(1511): LOCK_EVENT_WAIT pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee 2024-11-10T11:03:10,728 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1e67176538e0c6b983734c0b73e922ed, regionState=CLOSING, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:10,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, UNASSIGN because future has completed 2024-11-10T11:03:10,731 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-10T11:03:10,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e67176538e0c6b983734c0b73e922ed, server=eccb6f743e07,36119,1731236575621}] 2024-11-10T11:03:10,889 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(122): Close 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,889 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-10T11:03:10,890 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1722): Closing 1e67176538e0c6b983734c0b73e922ed, disabling compactions & flushes 2024-11-10T11:03:10,890 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:10,890 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:10,890 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. after waiting 0 ms 2024-11-10T11:03:10,890 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:10,890 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(2902): Flushing 1e67176538e0c6b983734c0b73e922ed 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-10T11:03:10,894 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/55e866048d3e41a9ab1dfc3ea410e442 is 1080, key is row0094/info:/1731236590686/Put/seqid=0 2024-11-10T11:03:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741846_1022 (size=8193) 2024-11-10T11:03:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741846_1022 (size=8193) 2024-11-10T11:03:10,900 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=123 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/55e866048d3e41a9ab1dfc3ea410e442 2024-11-10T11:03:10,905 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/.tmp/info/55e866048d3e41a9ab1dfc3ea410e442 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/55e866048d3e41a9ab1dfc3ea410e442 2024-11-10T11:03:10,910 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/55e866048d3e41a9ab1dfc3ea410e442, entries=3, sequenceid=123, filesize=8.0 K 2024-11-10T11:03:10,911 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1e67176538e0c6b983734c0b73e922ed in 21ms, sequenceid=123, compaction requested=true 2024-11-10T11:03:10,912 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/906aa1596cc944f5b0be4f829d21d352, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/33215cc55cd04898b171cd9c1a52f15a, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/b636c4e51d464912bbda151b0ad8ff82, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/7b6f2d60894f46f6835e76ab5629e730] to archive 2024-11-10T11:03:10,912 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T11:03:10,914 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/906aa1596cc944f5b0be4f829d21d352 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/906aa1596cc944f5b0be4f829d21d352 2024-11-10T11:03:10,915 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/ad86411152754dc9817ce8cbe442f1e0 2024-11-10T11:03:10,916 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/8d77e0caad064ed7a1ae3116ab09dbb1 2024-11-10T11:03:10,917 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/33215cc55cd04898b171cd9c1a52f15a to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/33215cc55cd04898b171cd9c1a52f15a 2024-11-10T11:03:10,918 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/b636c4e51d464912bbda151b0ad8ff82 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/b636c4e51d464912bbda151b0ad8ff82 2024-11-10T11:03:10,919 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/7b6f2d60894f46f6835e76ab5629e730 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/7b6f2d60894f46f6835e76ab5629e730 2024-11-10T11:03:10,924 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=1 2024-11-10T11:03:10,925 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. 2024-11-10T11:03:10,925 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] regionserver.HRegion(1676): Region close journal for 1e67176538e0c6b983734c0b73e922ed: Waiting for close lock at 1731236590890Running coprocessor pre-close hooks at 1731236590890Disabling compacts and flushes for region at 1731236590890Disabling writes for close at 1731236590890Obtaining lock to block concurrent updates at 1731236590890Preparing flush snapshotting stores in 1e67176538e0c6b983734c0b73e922ed at 1731236590890Finished memstore snapshotting TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1731236590890Flushing stores of TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. at 1731236590891 (+1 ms)Flushing 1e67176538e0c6b983734c0b73e922ed/info: creating writer at 1731236590891Flushing 1e67176538e0c6b983734c0b73e922ed/info: appending metadata at 1731236590894 (+3 ms)Flushing 1e67176538e0c6b983734c0b73e922ed/info: closing flushed file at 1731236590894Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c0c6f0: reopening flushed file at 1731236590904 (+10 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1e67176538e0c6b983734c0b73e922ed in 21ms, sequenceid=123, compaction requested=true at 1731236590911 (+7 ms)Writing region close event to WAL at 1731236590921 (+10 ms)Running coprocessor post-close hooks at 1731236590925 (+4 ms)Closed at 1731236590925 2024-11-10T11:03:10,927 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION, pid=10}] handler.UnassignRegionHandler(157): Closed 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,928 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=1e67176538e0c6b983734c0b73e922ed, regionState=CLOSED 2024-11-10T11:03:10,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=9, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1e67176538e0c6b983734c0b73e922ed, server=eccb6f743e07,36119,1731236575621 because future has completed 2024-11-10T11:03:10,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T11:03:10,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; CloseRegionProcedure 1e67176538e0c6b983734c0b73e922ed, server=eccb6f743e07,36119,1731236575621 in 200 msec 2024-11-10T11:03:10,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-10T11:03:10,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1e67176538e0c6b983734c0b73e922ed, UNASSIGN in 212 msec 2024-11-10T11:03:10,943 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:10,946 INFO [PEWorker-5 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 4 storefiles, region=1e67176538e0c6b983734c0b73e922ed, threads=4 2024-11-10T11:03:10,948 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/3f29ecfe653449d9a0acdb54d47e45f7 for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,948 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/f11f2fd444a64c27aa5386c6e4de6e3b for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,948 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730 for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,948 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/55e866048d3e41a9ab1dfc3ea410e442 for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,958 DEBUG [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/f11f2fd444a64c27aa5386c6e4de6e3b, top=true 2024-11-10T11:03:10,959 DEBUG [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/55e866048d3e41a9ab1dfc3ea410e442, top=true 2024-11-10T11:03:10,962 DEBUG [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(650): Will create HFileLink file for hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/3f29ecfe653449d9a0acdb54d47e45f7, top=true 2024-11-10T11:03:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741847_1023 (size=27) 2024-11-10T11:03:10,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741847_1023 (size=27) 2024-11-10T11:03:10,977 INFO [StoreFileSplitter-pool-3 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b for child: 4969bd98a51c96ecbb473e3472dc6531, parent: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,977 DEBUG [StoreFileSplitter-pool-3 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/f11f2fd444a64c27aa5386c6e4de6e3b for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,981 INFO [StoreFileSplitter-pool-2 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442 for child: 4969bd98a51c96ecbb473e3472dc6531, parent: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,981 DEBUG [StoreFileSplitter-pool-2 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/55e866048d3e41a9ab1dfc3ea410e442 for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,981 INFO [StoreFileSplitter-pool-1 {}] regionserver.HRegionFileSystem(691): Created linkFile:hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7 for child: 4969bd98a51c96ecbb473e3472dc6531, parent: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,981 DEBUG [StoreFileSplitter-pool-1 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/3f29ecfe653449d9a0acdb54d47e45f7 for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741848_1024 (size=27) 2024-11-10T11:03:10,986 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730 for region: 1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:10,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741848_1024 (size=27) 2024-11-10T11:03:10,989 DEBUG [PEWorker-5 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 1e67176538e0c6b983734c0b73e922ed Daughter A: [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed] storefiles, Daughter B: [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b] storefiles. 2024-11-10T11:03:10,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741849_1025 (size=71) 2024-11-10T11:03:10,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741849_1025 (size=71) 2024-11-10T11:03:11,000 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:11,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741850_1026 (size=71) 2024-11-10T11:03:11,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741850_1026 (size=71) 2024-11-10T11:03:11,012 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:11,023 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1 2024-11-10T11:03:11,025 DEBUG [PEWorker-5 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/recovered.edits/126.seqid, newMaxSeqId=126, maxSeqId=-1 2024-11-10T11:03:11,028 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1731236591027"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1731236591027"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1731236591027"}]},"ts":"1731236591027"} 2024-11-10T11:03:11,028 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731236591027"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236591027"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731236591027"}]},"ts":"1731236591027"} 2024-11-10T11:03:11,028 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1731236591027"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731236591027"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1731236591027"}]},"ts":"1731236591027"} 2024-11-10T11:03:11,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6dc8a2fd92092c8fc02930629a7fa4ce, ASSIGN}, {pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4969bd98a51c96ecbb473e3472dc6531, ASSIGN}] 2024-11-10T11:03:11,046 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6dc8a2fd92092c8fc02930629a7fa4ce, ASSIGN 2024-11-10T11:03:11,046 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4969bd98a51c96ecbb473e3472dc6531, ASSIGN 2024-11-10T11:03:11,047 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6dc8a2fd92092c8fc02930629a7fa4ce, ASSIGN; state=SPLITTING_NEW, location=eccb6f743e07,36119,1731236575621; forceNewPlan=false, retain=false 2024-11-10T11:03:11,047 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4969bd98a51c96ecbb473e3472dc6531, ASSIGN; state=SPLITTING_NEW, location=eccb6f743e07,36119,1731236575621; forceNewPlan=false, retain=false 2024-11-10T11:03:11,197 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6dc8a2fd92092c8fc02930629a7fa4ce, regionState=OPENING, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:11,198 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=4969bd98a51c96ecbb473e3472dc6531, regionState=OPENING, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:11,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6dc8a2fd92092c8fc02930629a7fa4ce, ASSIGN because future has completed 2024-11-10T11:03:11,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dc8a2fd92092c8fc02930629a7fa4ce, server=eccb6f743e07,36119,1731236575621}] 2024-11-10T11:03:11,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4969bd98a51c96ecbb473e3472dc6531, ASSIGN because future has completed 2024-11-10T11:03:11,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621}] 2024-11-10T11:03:11,355 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:11,356 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 6dc8a2fd92092c8fc02930629a7fa4ce, NAME => 'TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.', STARTKEY => '', ENDKEY => 'row0062'} 2024-11-10T11:03:11,356 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,356 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:03:11,356 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,356 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,357 INFO [StoreOpener-6dc8a2fd92092c8fc02930629a7fa4ce-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,358 INFO [StoreOpener-6dc8a2fd92092c8fc02930629a7fa4ce-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6dc8a2fd92092c8fc02930629a7fa4ce columnFamilyName info 2024-11-10T11:03:11,358 DEBUG [StoreOpener-6dc8a2fd92092c8fc02930629a7fa4ce-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:11,368 DEBUG [StoreOpener-6dc8a2fd92092c8fc02930629a7fa4ce-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed->hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730-bottom 2024-11-10T11:03:11,369 INFO [StoreOpener-6dc8a2fd92092c8fc02930629a7fa4ce-1 {}] regionserver.HStore(327): Store=6dc8a2fd92092c8fc02930629a7fa4ce/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:03:11,369 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,370 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,371 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,371 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,371 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,373 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,374 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 6dc8a2fd92092c8fc02930629a7fa4ce; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=689739, jitterRate=-0.12295269966125488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T11:03:11,374 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:11,374 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 6dc8a2fd92092c8fc02930629a7fa4ce: Running coprocessor pre-open hook at 1731236591356Writing region info on filesystem at 1731236591356Initializing all the Stores at 1731236591357 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236591357Cleaning up temporary data from old regions at 1731236591371 (+14 ms)Running coprocessor post-open hooks at 1731236591374 (+3 ms)Region opened successfully at 1731236591374 2024-11-10T11:03:11,375 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce., pid=13, masterSystemTime=1731236591352 2024-11-10T11:03:11,375 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 6dc8a2fd92092c8fc02930629a7fa4ce:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:11,375 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:11,375 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-10T11:03:11,376 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:11,376 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 6dc8a2fd92092c8fc02930629a7fa4ce/info is initiating minor compaction (all files) 2024-11-10T11:03:11,376 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6dc8a2fd92092c8fc02930629a7fa4ce/info in TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:11,376 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed->hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730-bottom] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/.tmp, totalSize=73.6 K 2024-11-10T11:03:11,377 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731236586523 2024-11-10T11:03:11,377 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:11,377 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:11,377 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:11,378 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7752): Opening region: {ENCODED => 4969bd98a51c96ecbb473e3472dc6531, NAME => 'TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.', STARTKEY => 'row0062', ENDKEY => ''} 2024-11-10T11:03:11,378 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,378 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:03:11,378 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7794): checking encryption for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,378 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(7797): checking classloading for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,378 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=6dc8a2fd92092c8fc02930629a7fa4ce, regionState=OPEN, openSeqNum=127, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:11,379 INFO [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,380 INFO [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4969bd98a51c96ecbb473e3472dc6531 columnFamilyName info 2024-11-10T11:03:11,380 DEBUG [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:11,380 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-10T11:03:11,380 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-10T11:03:11,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-11-10T11:03:11,380 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dc8a2fd92092c8fc02930629a7fa4ce, server=eccb6f743e07,36119,1731236575621 because future has completed 2024-11-10T11:03:11,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11 2024-11-10T11:03:11,384 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 6dc8a2fd92092c8fc02930629a7fa4ce, server=eccb6f743e07,36119,1731236575621 in 180 msec 2024-11-10T11:03:11,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=6dc8a2fd92092c8fc02930629a7fa4ce, ASSIGN in 339 msec 2024-11-10T11:03:11,392 DEBUG [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed->hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730-top 2024-11-10T11:03:11,397 DEBUG [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7 2024-11-10T11:03:11,398 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6dc8a2fd92092c8fc02930629a7fa4ce#info#compaction#64 average throughput is 20.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:11,398 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/.tmp/info/2924affd7c0b496096571f8266fddfc3 is 1080, key is row0001/info:/1731236586523/Put/seqid=0 2024-11-10T11:03:11,402 DEBUG [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442 2024-11-10T11:03:11,405 DEBUG [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b 2024-11-10T11:03:11,405 INFO [StoreOpener-4969bd98a51c96ecbb473e3472dc6531-1 {}] regionserver.HStore(327): Store=4969bd98a51c96ecbb473e3472dc6531/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:03:11,405 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1038): replaying wal for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,406 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,407 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/info/e823696273fe46b797fdba79dcc1c034 is 193, key is TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531./info:regioninfo/1731236591197/Put/seqid=0 2024-11-10T11:03:11,407 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1048): stopping wal replay for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,407 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1060): Cleaning up temporary data for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,409 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1093): writing seq id for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741851_1027 (size=70862) 2024-11-10T11:03:11,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741851_1027 (size=70862) 2024-11-10T11:03:11,410 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1114): Opened 4969bd98a51c96ecbb473e3472dc6531; next sequenceid=127; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=823597, jitterRate=0.04725868999958038}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T11:03:11,410 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:11,410 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegion(1006): Region open journal for 4969bd98a51c96ecbb473e3472dc6531: Running coprocessor pre-open hook at 1731236591378Writing region info on filesystem at 1731236591378Initializing all the Stores at 1731236591379 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236591379Cleaning up temporary data from old regions at 1731236591407 (+28 ms)Running coprocessor post-open hooks at 1731236591410 (+3 ms)Region opened successfully at 1731236591410 2024-11-10T11:03:11,411 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., pid=14, masterSystemTime=1731236591352 2024-11-10T11:03:11,411 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 2 2024-11-10T11:03:11,411 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:11,411 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T11:03:11,413 INFO [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:11,413 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:11,413 INFO [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:11,413 INFO [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed->hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730-top, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=120.8 K 2024-11-10T11:03:11,414 DEBUG [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:11,414 INFO [RS_OPEN_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_REGION, pid=14}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:11,414 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] compactions.Compactor(225): Compacting 30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed, keycount=32, bloomtype=ROW, size=73.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731236586523 2024-11-10T11:03:11,414 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1731236590637 2024-11-10T11:03:11,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=4969bd98a51c96ecbb473e3472dc6531, regionState=OPEN, openSeqNum=127, regionLocation=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:11,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741852_1028 (size=9847) 2024-11-10T11:03:11,415 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731236590660 2024-11-10T11:03:11,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741852_1028 (size=9847) 2024-11-10T11:03:11,415 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] compactions.Compactor(225): Compacting TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731236590686 2024-11-10T11:03:11,416 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/.tmp/info/2924affd7c0b496096571f8266fddfc3 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/2924affd7c0b496096571f8266fddfc3 2024-11-10T11:03:11,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/info/e823696273fe46b797fdba79dcc1c034 2024-11-10T11:03:11,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621 because future has completed 2024-11-10T11:03:11,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=12 2024-11-10T11:03:11,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621 in 216 msec 2024-11-10T11:03:11,422 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 6dc8a2fd92092c8fc02930629a7fa4ce/info of 6dc8a2fd92092c8fc02930629a7fa4ce into 2924affd7c0b496096571f8266fddfc3(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:11,423 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6dc8a2fd92092c8fc02930629a7fa4ce: 2024-11-10T11:03:11,423 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce., storeName=6dc8a2fd92092c8fc02930629a7fa4ce/info, priority=15, startTime=1731236591375; duration=0sec 2024-11-10T11:03:11,423 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:11,423 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6dc8a2fd92092c8fc02930629a7fa4ce:info 2024-11-10T11:03:11,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=7 2024-11-10T11:03:11,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4969bd98a51c96ecbb473e3472dc6531, ASSIGN in 376 msec 2024-11-10T11:03:11,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=6dc8a2fd92092c8fc02930629a7fa4ce, daughterB=4969bd98a51c96ecbb473e3472dc6531 in 715 msec 2024-11-10T11:03:11,427 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee 2024-11-10T11:03:11,427 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee 2024-11-10T11:03:11,427 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee 2024-11-10T11:03:11,428 INFO [PEWorker-5 {}] assignment.SplitTableRegionProcedure(534): Split of {ENCODED => 1e67176538e0c6b983734c0b73e922ed, NAME => 'TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed.', STARTKEY => '', ENDKEY => ''} skipped; state is already SPLIT 2024-11-10T11:03:11,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1e67176538e0c6b983734c0b73e922ed, daughterA=73c053b51e1ba72ce67ea1af760edf5b, daughterB=34fcef7ffaeabdf8512db32e2b4c5bee in 712 msec 2024-11-10T11:03:11,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/ns/9b05e8c45fe6418ba6a680582f1fdbd5 is 43, key is default/ns:d/1731236576406/Put/seqid=0 2024-11-10T11:03:11,439 INFO [RS:0;eccb6f743e07:36119-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#67 average throughput is 35.92 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:11,440 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/eaef03b565984cc694d0396bbaa03553 is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:11,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741853_1029 (size=5153) 2024-11-10T11:03:11,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741853_1029 (size=5153) 2024-11-10T11:03:11,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/ns/9b05e8c45fe6418ba6a680582f1fdbd5 2024-11-10T11:03:11,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741854_1030 (size=43081) 2024-11-10T11:03:11,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741854_1030 (size=43081) 2024-11-10T11:03:11,449 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/eaef03b565984cc694d0396bbaa03553 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/eaef03b565984cc694d0396bbaa03553 2024-11-10T11:03:11,454 INFO [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into eaef03b565984cc694d0396bbaa03553(size=42.1 K), total size for store is 42.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:11,454 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:11,454 INFO [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=12, startTime=1731236591411; duration=0sec 2024-11-10T11:03:11,454 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:11,454 DEBUG [RS:0;eccb6f743e07:36119-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:11,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/table/77dc3f14cdfc4690a89d038405de95aa is 65, key is TestLogRolling-testLogRolling/table:state/1731236576811/Put/seqid=0 2024-11-10T11:03:11,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741855_1031 (size=5340) 2024-11-10T11:03:11,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741855_1031 (size=5340) 2024-11-10T11:03:11,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/table/77dc3f14cdfc4690a89d038405de95aa 2024-11-10T11:03:11,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/info/e823696273fe46b797fdba79dcc1c034 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/info/e823696273fe46b797fdba79dcc1c034 2024-11-10T11:03:11,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/info/e823696273fe46b797fdba79dcc1c034, entries=30, sequenceid=17, filesize=9.6 K 2024-11-10T11:03:11,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/ns/9b05e8c45fe6418ba6a680582f1fdbd5 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/ns/9b05e8c45fe6418ba6a680582f1fdbd5 2024-11-10T11:03:11,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/ns/9b05e8c45fe6418ba6a680582f1fdbd5, entries=2, sequenceid=17, filesize=5.0 K 2024-11-10T11:03:11,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/table/77dc3f14cdfc4690a89d038405de95aa as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/table/77dc3f14cdfc4690a89d038405de95aa 2024-11-10T11:03:11,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/table/77dc3f14cdfc4690a89d038405de95aa, entries=2, sequenceid=17, filesize=5.2 K 2024-11-10T11:03:11,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 109ms, sequenceid=17, compaction requested=false 2024-11-10T11:03:11,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T11:03:11,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:11,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:12,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:12,612 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60922 deadline: 1731236602691, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. is not online on eccb6f743e07,36119,1731236575621 2024-11-10T11:03:12,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., hostname=eccb6f743e07,36119,1731236575621, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., hostname=eccb6f743e07,36119,1731236575621, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. is not online on eccb6f743e07,36119,1731236575621 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T11:03:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., hostname=eccb6f743e07,36119,1731236575621, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed. is not online on eccb6f743e07,36119,1731236575621 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T11:03:12,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1731236576451.1e67176538e0c6b983734c0b73e922ed., hostname=eccb6f743e07,36119,1731236575621, seqNum=2 from cache 2024-11-10T11:03:13,613 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:13,613 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:14,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:14,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:15,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:15,614 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:15,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,926 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,927 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,927 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,950 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,950 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,951 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,955 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,955 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,955 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:15,959 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-10T11:03:16,467 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,467 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,467 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,467 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,467 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,468 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,468 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,468 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,488 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,489 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,490 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,491 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,496 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,496 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,497 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,501 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-10T11:03:16,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:16,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:17,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:17,615 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:18,616 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:18,616 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:19,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:19,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:20,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:20,617 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:21,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:21,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:22,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:22,618 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:22,811 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0097', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., hostname=eccb6f743e07,36119,1731236575621, seqNum=127] 2024-11-10T11:03:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:22,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:03:22,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/987ec16b1bba4aa29834728bcaef3da7 is 1080, key is row0097/info:/1731236602812/Put/seqid=0 2024-11-10T11:03:22,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741856_1032 (size=12516) 2024-11-10T11:03:22,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741856_1032 (size=12516) 2024-11-10T11:03:22,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/987ec16b1bba4aa29834728bcaef3da7 2024-11-10T11:03:22,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/987ec16b1bba4aa29834728bcaef3da7 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/987ec16b1bba4aa29834728bcaef3da7 2024-11-10T11:03:22,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/987ec16b1bba4aa29834728bcaef3da7, entries=7, sequenceid=137, filesize=12.2 K 2024-11-10T11:03:22,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 4969bd98a51c96ecbb473e3472dc6531 in 22ms, sequenceid=137, compaction requested=false 2024-11-10T11:03:22,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:22,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T11:03:22,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/ba0683275623423dbd0e5f8357822197 is 1080, key is row0104/info:/1731236602823/Put/seqid=0 2024-11-10T11:03:22,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741857_1033 (size=21156) 2024-11-10T11:03:22,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741857_1033 (size=21156) 2024-11-10T11:03:22,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/ba0683275623423dbd0e5f8357822197 2024-11-10T11:03:22,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/ba0683275623423dbd0e5f8357822197 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/ba0683275623423dbd0e5f8357822197 2024-11-10T11:03:22,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/ba0683275623423dbd0e5f8357822197, entries=15, sequenceid=155, filesize=20.7 K 2024-11-10T11:03:22,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=10.51 KB/10760 for 4969bd98a51c96ecbb473e3472dc6531 in 25ms, sequenceid=155, compaction requested=true 2024-11-10T11:03:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:22,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:22,869 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:22,870 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 76753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:22,870 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:22,870 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:22,870 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/eaef03b565984cc694d0396bbaa03553, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/987ec16b1bba4aa29834728bcaef3da7, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/ba0683275623423dbd0e5f8357822197] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=75.0 K 2024-11-10T11:03:22,871 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaef03b565984cc694d0396bbaa03553, keycount=35, bloomtype=ROW, size=42.1 K, encoding=NONE, compression=NONE, seqNum=123, earliestPutTs=1731236588627 2024-11-10T11:03:22,871 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 987ec16b1bba4aa29834728bcaef3da7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731236602812 2024-11-10T11:03:22,871 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba0683275623423dbd0e5f8357822197, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731236602823 2024-11-10T11:03:22,881 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#71 average throughput is 58.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:22,881 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a91844cc7eee4c96b5734ee4a6ab0e72 is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:22,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741858_1034 (size=66967) 2024-11-10T11:03:22,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741858_1034 (size=66967) 2024-11-10T11:03:22,891 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a91844cc7eee4c96b5734ee4a6ab0e72 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a91844cc7eee4c96b5734ee4a6ab0e72 2024-11-10T11:03:22,896 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into a91844cc7eee4c96b5734ee4a6ab0e72(size=65.4 K), total size for store is 65.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:22,896 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:22,896 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=13, startTime=1731236602869; duration=0sec 2024-11-10T11:03:22,896 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:22,896 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:23,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:23,619 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:24,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:24,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:24,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-10T11:03:24,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/83dfb19d93e0423b89737e7ef3be08c0 is 1080, key is row0119/info:/1731236602845/Put/seqid=0 2024-11-10T11:03:24,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741859_1035 (size=16828) 2024-11-10T11:03:24,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741859_1035 (size=16828) 2024-11-10T11:03:24,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/83dfb19d93e0423b89737e7ef3be08c0 2024-11-10T11:03:24,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/83dfb19d93e0423b89737e7ef3be08c0 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/83dfb19d93e0423b89737e7ef3be08c0 2024-11-10T11:03:24,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/83dfb19d93e0423b89737e7ef3be08c0, entries=11, sequenceid=170, filesize=16.4 K 2024-11-10T11:03:24,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=13.66 KB/13988 for 4969bd98a51c96ecbb473e3472dc6531 in 21ms, sequenceid=170, compaction requested=false 2024-11-10T11:03:24,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:24,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T11:03:24,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c576d4dab37944ddb8edebe1559cfdfa is 1080, key is row0130/info:/1731236604865/Put/seqid=0 2024-11-10T11:03:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741860_1036 (size=20078) 2024-11-10T11:03:24,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741860_1036 (size=20078) 2024-11-10T11:03:24,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c576d4dab37944ddb8edebe1559cfdfa 2024-11-10T11:03:24,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c576d4dab37944ddb8edebe1559cfdfa as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c576d4dab37944ddb8edebe1559cfdfa 2024-11-10T11:03:24,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c576d4dab37944ddb8edebe1559cfdfa, entries=14, sequenceid=187, filesize=19.6 K 2024-11-10T11:03:24,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 4969bd98a51c96ecbb473e3472dc6531 in 21ms, sequenceid=187, compaction requested=true 2024-11-10T11:03:24,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:24,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:24,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:24,906 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:24,907 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103873 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:24,907 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:24,907 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:24,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T11:03:24,907 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a91844cc7eee4c96b5734ee4a6ab0e72, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/83dfb19d93e0423b89737e7ef3be08c0, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c576d4dab37944ddb8edebe1559cfdfa] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=101.4 K 2024-11-10T11:03:24,908 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a91844cc7eee4c96b5734ee4a6ab0e72, keycount=57, bloomtype=ROW, size=65.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731236588627 2024-11-10T11:03:24,908 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83dfb19d93e0423b89737e7ef3be08c0, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731236602845 2024-11-10T11:03:24,909 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c576d4dab37944ddb8edebe1559cfdfa, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731236604865 2024-11-10T11:03:24,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/dc36d066a9d6404595c1a689b49431ea is 1080, key is row0144/info:/1731236604886/Put/seqid=0 2024-11-10T11:03:24,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741861_1037 (size=20078) 2024-11-10T11:03:24,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741861_1037 (size=20078) 2024-11-10T11:03:24,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/dc36d066a9d6404595c1a689b49431ea 2024-11-10T11:03:24,920 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#75 average throughput is 42.07 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:24,921 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/e89117f1e8d24debaaf5ca9d65978a31 is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:24,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/dc36d066a9d6404595c1a689b49431ea as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/dc36d066a9d6404595c1a689b49431ea 2024-11-10T11:03:24,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741862_1038 (size=94096) 2024-11-10T11:03:24,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741862_1038 (size=94096) 2024-11-10T11:03:24,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/dc36d066a9d6404595c1a689b49431ea, entries=14, sequenceid=204, filesize=19.6 K 2024-11-10T11:03:24,932 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/e89117f1e8d24debaaf5ca9d65978a31 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/e89117f1e8d24debaaf5ca9d65978a31 2024-11-10T11:03:24,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=3.15 KB/3228 for 4969bd98a51c96ecbb473e3472dc6531 in 25ms, sequenceid=204, compaction requested=false 2024-11-10T11:03:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:24,938 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into e89117f1e8d24debaaf5ca9d65978a31(size=91.9 K), total size for store is 111.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:24,938 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:24,938 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=13, startTime=1731236604906; duration=0sec 2024-11-10T11:03:24,938 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:24,938 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:25,553 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T11:03:25,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:25,620 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:26,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:26,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:26,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:26,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:03:26,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c32c14948eb9450b9bdf04e2a32a1c45 is 1080, key is row0158/info:/1731236604908/Put/seqid=0 2024-11-10T11:03:26,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741863_1039 (size=12516) 2024-11-10T11:03:26,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741863_1039 (size=12516) 2024-11-10T11:03:26,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c32c14948eb9450b9bdf04e2a32a1c45 2024-11-10T11:03:26,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c32c14948eb9450b9bdf04e2a32a1c45 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c32c14948eb9450b9bdf04e2a32a1c45 2024-11-10T11:03:26,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c32c14948eb9450b9bdf04e2a32a1c45, entries=7, sequenceid=215, filesize=12.2 K 2024-11-10T11:03:26,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 4969bd98a51c96ecbb473e3472dc6531 in 38ms, sequenceid=215, compaction requested=true 2024-11-10T11:03:26,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:26,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:26,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:26,958 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:26,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB 2024-11-10T11:03:26,959 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 126690 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:26,959 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:26,959 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:26,960 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/e89117f1e8d24debaaf5ca9d65978a31, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/dc36d066a9d6404595c1a689b49431ea, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c32c14948eb9450b9bdf04e2a32a1c45] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=123.7 K 2024-11-10T11:03:26,960 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting e89117f1e8d24debaaf5ca9d65978a31, keycount=82, bloomtype=ROW, size=91.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731236588627 2024-11-10T11:03:26,961 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc36d066a9d6404595c1a689b49431ea, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731236604886 2024-11-10T11:03:26,961 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c32c14948eb9450b9bdf04e2a32a1c45, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731236604908 2024-11-10T11:03:26,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/7976c6e586514a8aa8ba5a1c9b3ec9de is 1080, key is row0165/info:/1731236606920/Put/seqid=0 2024-11-10T11:03:26,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741864_1040 (size=24394) 2024-11-10T11:03:26,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741864_1040 (size=24394) 2024-11-10T11:03:26,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/7976c6e586514a8aa8ba5a1c9b3ec9de 2024-11-10T11:03:26,975 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#78 average throughput is 52.85 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:26,975 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/1d6b42e273334fb392a635e16cdc1299 is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/7976c6e586514a8aa8ba5a1c9b3ec9de as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7976c6e586514a8aa8ba5a1c9b3ec9de 2024-11-10T11:03:26,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741865_1041 (size=116840) 2024-11-10T11:03:26,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741865_1041 (size=116840) 2024-11-10T11:03:26,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7976c6e586514a8aa8ba5a1c9b3ec9de, entries=18, sequenceid=236, filesize=23.8 K 2024-11-10T11:03:26,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=10.51 KB/10760 for 4969bd98a51c96ecbb473e3472dc6531 in 26ms, sequenceid=236, compaction requested=false 2024-11-10T11:03:26,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:26,986 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/1d6b42e273334fb392a635e16cdc1299 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/1d6b42e273334fb392a635e16cdc1299 2024-11-10T11:03:26,991 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into 1d6b42e273334fb392a635e16cdc1299(size=114.1 K), total size for store is 137.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:26,991 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:26,991 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=13, startTime=1731236606958; duration=0sec 2024-11-10T11:03:26,991 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:26,991 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:27,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:27,621 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:28,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:28,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:28,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-10T11:03:28,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a0ce2a50d96049de858364ff8d0428af is 1080, key is row0183/info:/1731236606960/Put/seqid=0 2024-11-10T11:03:28,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741866_1042 (size=16828) 2024-11-10T11:03:28,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741866_1042 (size=16828) 2024-11-10T11:03:28,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a0ce2a50d96049de858364ff8d0428af 2024-11-10T11:03:29,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a0ce2a50d96049de858364ff8d0428af as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a0ce2a50d96049de858364ff8d0428af 2024-11-10T11:03:29,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a0ce2a50d96049de858364ff8d0428af, entries=11, sequenceid=251, filesize=16.4 K 2024-11-10T11:03:29,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=15.76 KB/16140 for 4969bd98a51c96ecbb473e3472dc6531 in 23ms, sequenceid=251, compaction requested=true 2024-11-10T11:03:29,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:29,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:29,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:29,007 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:29,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=16.81 KB heapSize=18.25 KB 2024-11-10T11:03:29,008 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 158062 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:29,008 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:29,008 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:29,009 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/1d6b42e273334fb392a635e16cdc1299, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7976c6e586514a8aa8ba5a1c9b3ec9de, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a0ce2a50d96049de858364ff8d0428af] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=154.4 K 2024-11-10T11:03:29,009 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6b42e273334fb392a635e16cdc1299, keycount=103, bloomtype=ROW, size=114.1 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731236588627 2024-11-10T11:03:29,009 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7976c6e586514a8aa8ba5a1c9b3ec9de, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731236606920 2024-11-10T11:03:29,010 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0ce2a50d96049de858364ff8d0428af, keycount=11, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731236606960 2024-11-10T11:03:29,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/72e240fd8299464e86286a158510472d is 1080, key is row0194/info:/1731236608985/Put/seqid=0 2024-11-10T11:03:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741867_1043 (size=22251) 2024-11-10T11:03:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741867_1043 (size=22251) 2024-11-10T11:03:29,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.81 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/72e240fd8299464e86286a158510472d 2024-11-10T11:03:29,025 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#81 average throughput is 45.15 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:29,026 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a984714cf52b4ac6ba4bfe046d59905c is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:29,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/72e240fd8299464e86286a158510472d as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/72e240fd8299464e86286a158510472d 2024-11-10T11:03:29,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741868_1044 (size=148409) 2024-11-10T11:03:29,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741868_1044 (size=148409) 2024-11-10T11:03:29,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T11:03:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:60922 deadline: 1731236619034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621 2024-11-10T11:03:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., hostname=eccb6f743e07,36119,1731236575621, seqNum=127 , the old value is region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., hostname=eccb6f743e07,36119,1731236575621, seqNum=127, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T11:03:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., hostname=eccb6f743e07,36119,1731236575621, seqNum=127 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4969bd98a51c96ecbb473e3472dc6531, server=eccb6f743e07,36119,1731236575621 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T11:03:29,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., hostname=eccb6f743e07,36119,1731236575621, seqNum=127 because the exception is null or not the one we care about 2024-11-10T11:03:29,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/72e240fd8299464e86286a158510472d, entries=16, sequenceid=270, filesize=21.7 K 2024-11-10T11:03:29,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~16.81 KB/17216, heapSize ~18.23 KB/18672, currentSize=13.66 KB/13988 for 4969bd98a51c96ecbb473e3472dc6531 in 30ms, sequenceid=270, compaction requested=false 2024-11-10T11:03:29,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:29,039 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/a984714cf52b4ac6ba4bfe046d59905c as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a984714cf52b4ac6ba4bfe046d59905c 2024-11-10T11:03:29,045 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into a984714cf52b4ac6ba4bfe046d59905c(size=144.9 K), total size for store is 166.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:29,045 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:29,045 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=13, startTime=1731236609007; duration=0sec 2024-11-10T11:03:29,045 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:29,045 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:29,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:29,622 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:30,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:30,623 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:31,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:31,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:32,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:32,624 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:33,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:33,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:34,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:34,625 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:35,626 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:35,626 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:36,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:36,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:36,654 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T11:03:36,654 INFO [master/eccb6f743e07:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T11:03:37,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:37,627 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:38,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:38,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:39,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-10T11:03:39,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/68a2f457bacc489d99a8869adf102ad0 is 1080, key is row0210/info:/1731236609009/Put/seqid=0 2024-11-10T11:03:39,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741869_1045 (size=20092) 2024-11-10T11:03:39,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741869_1045 (size=20092) 2024-11-10T11:03:39,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/68a2f457bacc489d99a8869adf102ad0 2024-11-10T11:03:39,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/68a2f457bacc489d99a8869adf102ad0 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/68a2f457bacc489d99a8869adf102ad0 2024-11-10T11:03:39,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/68a2f457bacc489d99a8869adf102ad0, entries=14, sequenceid=288, filesize=19.6 K 2024-11-10T11:03:39,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=1.05 KB/1076 for 4969bd98a51c96ecbb473e3472dc6531 in 21ms, sequenceid=288, compaction requested=true 2024-11-10T11:03:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:39,119 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:39,120 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:39,120 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 190752 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:39,121 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:39,121 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:39,121 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a984714cf52b4ac6ba4bfe046d59905c, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/72e240fd8299464e86286a158510472d, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/68a2f457bacc489d99a8869adf102ad0] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=186.3 K 2024-11-10T11:03:39,121 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting a984714cf52b4ac6ba4bfe046d59905c, keycount=132, bloomtype=ROW, size=144.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1731236588627 2024-11-10T11:03:39,121 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e240fd8299464e86286a158510472d, keycount=16, bloomtype=ROW, size=21.7 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731236608985 2024-11-10T11:03:39,122 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68a2f457bacc489d99a8869adf102ad0, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731236609009 2024-11-10T11:03:39,132 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#83 average throughput is 83.12 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:39,133 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/75fa1698c12a4213bdef41c2e421c5fd is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741870_1046 (size=180886) 2024-11-10T11:03:39,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741870_1046 (size=180886) 2024-11-10T11:03:39,142 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/75fa1698c12a4213bdef41c2e421c5fd as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/75fa1698c12a4213bdef41c2e421c5fd 2024-11-10T11:03:39,147 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into 75fa1698c12a4213bdef41c2e421c5fd(size=176.6 K), total size for store is 176.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:39,147 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:39,147 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=13, startTime=1731236619119; duration=0sec 2024-11-10T11:03:39,147 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:39,147 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:39,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:39,628 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:40,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:40,629 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:41,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-10T11:03:41,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c8593ccec9844437aeb0cb935344854e is 1080, key is row0224/info:/1731236619099/Put/seqid=0 2024-11-10T11:03:41,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741871_1047 (size=12523) 2024-11-10T11:03:41,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741871_1047 (size=12523) 2024-11-10T11:03:41,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c8593ccec9844437aeb0cb935344854e 2024-11-10T11:03:41,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/c8593ccec9844437aeb0cb935344854e as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c8593ccec9844437aeb0cb935344854e 2024-11-10T11:03:41,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c8593ccec9844437aeb0cb935344854e, entries=7, sequenceid=299, filesize=12.2 K 2024-11-10T11:03:41,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 4969bd98a51c96ecbb473e3472dc6531 in 22ms, sequenceid=299, compaction requested=false 2024-11-10T11:03:41,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36119 {}] regionserver.HRegion(8855): Flush requested on 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:41,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-10T11:03:41,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/bfc0676d76b84b0eb8957f991cdd1372 is 1080, key is row0231/info:/1731236621110/Put/seqid=0 2024-11-10T11:03:41,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741872_1048 (size=21171) 2024-11-10T11:03:41,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741872_1048 (size=21171) 2024-11-10T11:03:41,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/bfc0676d76b84b0eb8957f991cdd1372 2024-11-10T11:03:41,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/bfc0676d76b84b0eb8957f991cdd1372 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/bfc0676d76b84b0eb8957f991cdd1372 2024-11-10T11:03:41,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/bfc0676d76b84b0eb8957f991cdd1372, entries=15, sequenceid=317, filesize=20.7 K 2024-11-10T11:03:41,154 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=11.56 KB/11836 for 4969bd98a51c96ecbb473e3472dc6531 in 22ms, sequenceid=317, compaction requested=true 2024-11-10T11:03:41,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:41,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4969bd98a51c96ecbb473e3472dc6531:info, priority=-2147483648, current under compaction store size is 1 2024-11-10T11:03:41,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:41,154 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T11:03:41,155 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 214580 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T11:03:41,155 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1541): 4969bd98a51c96ecbb473e3472dc6531/info is initiating minor compaction (all files) 2024-11-10T11:03:41,155 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4969bd98a51c96ecbb473e3472dc6531/info in TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:41,155 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/75fa1698c12a4213bdef41c2e421c5fd, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c8593ccec9844437aeb0cb935344854e, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/bfc0676d76b84b0eb8957f991cdd1372] into tmpdir=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp, totalSize=209.6 K 2024-11-10T11:03:41,155 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75fa1698c12a4213bdef41c2e421c5fd, keycount=162, bloomtype=ROW, size=176.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731236588627 2024-11-10T11:03:41,156 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8593ccec9844437aeb0cb935344854e, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731236619099 2024-11-10T11:03:41,156 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfc0676d76b84b0eb8957f991cdd1372, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731236621110 2024-11-10T11:03:41,166 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4969bd98a51c96ecbb473e3472dc6531#info#compaction#86 average throughput is 62.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T11:03:41,167 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/7eb3ec2e8da94ba898aca82ee0627217 is 1080, key is row0062/info:/1731236588627/Put/seqid=0 2024-11-10T11:03:41,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741873_1049 (size=204803) 2024-11-10T11:03:41,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741873_1049 (size=204803) 2024-11-10T11:03:41,175 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/7eb3ec2e8da94ba898aca82ee0627217 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7eb3ec2e8da94ba898aca82ee0627217 2024-11-10T11:03:41,180 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4969bd98a51c96ecbb473e3472dc6531/info of 4969bd98a51c96ecbb473e3472dc6531 into 7eb3ec2e8da94ba898aca82ee0627217(size=200.0 K), total size for store is 200.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T11:03:41,180 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:41,180 INFO [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531., storeName=4969bd98a51c96ecbb473e3472dc6531/info, priority=13, startTime=1731236621154; duration=0sec 2024-11-10T11:03:41,180 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T11:03:41,180 DEBUG [RS:0;eccb6f743e07:36119-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4969bd98a51c96ecbb473e3472dc6531:info 2024-11-10T11:03:41,377 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340 2024-11-10T11:03:41,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:41,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:42,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:42,630 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:43,149 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-10T11:03:43,150 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36119%2C1731236575621.1731236623150 2024-11-10T11:03:43,156 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,156 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,156 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,156 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,157 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,157 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236576000 with entries=310, filesize=307.89 KB; new WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236623150 2024-11-10T11:03:43,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741833_1009 (size=315283) 2024-11-10T11:03:43,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741833_1009 (size=315283) 2024-11-10T11:03:43,164 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39541:39541),(127.0.0.1/127.0.0.1:43017:43017)] 2024-11-10T11:03:43,167 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6dc8a2fd92092c8fc02930629a7fa4ce: 2024-11-10T11:03:43,167 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-11-10T11:03:43,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/info/ba6cae5d9cbf4e9fa94de3e454b30352 is 193, key is TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531./info:regioninfo/1731236591414/Put/seqid=0 2024-11-10T11:03:43,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741875_1051 (size=6223) 2024-11-10T11:03:43,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741875_1051 (size=6223) 2024-11-10T11:03:43,178 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/info/ba6cae5d9cbf4e9fa94de3e454b30352 2024-11-10T11:03:43,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/.tmp/info/ba6cae5d9cbf4e9fa94de3e454b30352 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/info/ba6cae5d9cbf4e9fa94de3e454b30352 2024-11-10T11:03:43,187 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/info/ba6cae5d9cbf4e9fa94de3e454b30352, entries=5, sequenceid=21, filesize=6.1 K 2024-11-10T11:03:43,188 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false 2024-11-10T11:03:43,188 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-10T11:03:43,188 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4969bd98a51c96ecbb473e3472dc6531 1/1 column families, dataSize=11.56 KB heapSize=12.63 KB 2024-11-10T11:03:43,192 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/45772bf89a944ab78057e42158a1fa99 is 1080, key is row0246/info:/1731236621133/Put/seqid=0 2024-11-10T11:03:43,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741876_1052 (size=16839) 2024-11-10T11:03:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741876_1052 (size=16839) 2024-11-10T11:03:43,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.56 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/45772bf89a944ab78057e42158a1fa99 2024-11-10T11:03:43,200 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/.tmp/info/45772bf89a944ab78057e42158a1fa99 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/45772bf89a944ab78057e42158a1fa99 2024-11-10T11:03:43,204 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/45772bf89a944ab78057e42158a1fa99, entries=11, sequenceid=332, filesize=16.4 K 2024-11-10T11:03:43,205 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~11.56 KB/11836, heapSize ~12.61 KB/12912, currentSize=0 B/0 for 4969bd98a51c96ecbb473e3472dc6531 in 17ms, sequenceid=332, compaction requested=false 2024-11-10T11:03:43,205 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4969bd98a51c96ecbb473e3472dc6531: 2024-11-10T11:03:43,205 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C36119%2C1731236575621.1731236623205 2024-11-10T11:03:43,211 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,211 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,211 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,211 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,211 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,211 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236623150 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236623205 2024-11-10T11:03:43,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:43017:43017),(127.0.0.1/127.0.0.1:39541:39541)] 2024-11-10T11:03:43,212 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236623150 is not closed yet, will try archiving it next time 2024-11-10T11:03:43,212 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236576000 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/oldWALs/eccb6f743e07%2C36119%2C1731236575621.1731236576000 2024-11-10T11:03:43,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741874_1050 (size=731) 2024-11-10T11:03:43,213 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T11:03:43,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741874_1050 (size=731) 2024-11-10T11:03:43,214 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/WALs/eccb6f743e07,36119,1731236575621/eccb6f743e07%2C36119%2C1731236575621.1731236623150 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/oldWALs/eccb6f743e07%2C36119%2C1731236575621.1731236623150 2024-11-10T11:03:43,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:03:43,313 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:03:43,313 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:03:43,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:43,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:43,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:03:43,314 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:03:43,314 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=210270111, stopped=false 2024-11-10T11:03:43,314 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,42865,1731236575570 2024-11-10T11:03:43,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:03:43,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:03:43,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:43,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:43,316 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:03:43,316 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:03:43,316 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:03:43,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:43,316 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,36119,1731236575621' ***** 2024-11-10T11:03:43,316 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:03:43,316 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:03:43,317 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:03:43,317 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(3091): Received CLOSE for 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(3091): Received CLOSE for 4969bd98a51c96ecbb473e3472dc6531 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,36119,1731236575621 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:36119. 2024-11-10T11:03:43,317 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6dc8a2fd92092c8fc02930629a7fa4ce, disabling compactions & flushes 2024-11-10T11:03:43,317 DEBUG [RS:0;eccb6f743e07:36119 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:03:43,317 DEBUG [RS:0;eccb6f743e07:36119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:43,317 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:43,317 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:03:43,317 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:03:43,317 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. after waiting 0 ms 2024-11-10T11:03:43,318 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:43,318 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:03:43,318 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-10T11:03:43,318 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1325): Online Regions={6dc8a2fd92092c8fc02930629a7fa4ce=TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce., 1588230740=hbase:meta,,1.1588230740, 4969bd98a51c96ecbb473e3472dc6531=TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.} 2024-11-10T11:03:43,318 DEBUG [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4969bd98a51c96ecbb473e3472dc6531, 6dc8a2fd92092c8fc02930629a7fa4ce 2024-11-10T11:03:43,318 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:03:43,318 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:03:43,318 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:03:43,318 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:03:43,318 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:03:43,318 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed->hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730-bottom] to archive 2024-11-10T11:03:43,319 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T11:03:43,321 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:43,321 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=eccb6f743e07:42865 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-10T11:03:43,321 WARN [StoreCloser-TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-10T11:03:43,322 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-10T11:03:43,323 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:03:43,323 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:03:43,323 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236623318Running coprocessor pre-close hooks at 1731236623318Disabling compacts and flushes for region at 1731236623318Disabling writes for close at 1731236623318Writing region close event to WAL at 1731236623319 (+1 ms)Running coprocessor post-close hooks at 1731236623323 (+4 ms)Closed at 1731236623323 2024-11-10T11:03:43,323 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:03:43,325 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/6dc8a2fd92092c8fc02930629a7fa4ce/recovered.edits/131.seqid, newMaxSeqId=131, maxSeqId=126 2024-11-10T11:03:43,325 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:43,325 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6dc8a2fd92092c8fc02930629a7fa4ce: Waiting for close lock at 1731236623317Running coprocessor pre-close hooks at 1731236623317Disabling compacts and flushes for region at 1731236623317Disabling writes for close at 1731236623318 (+1 ms)Writing region close event to WAL at 1731236623322 (+4 ms)Running coprocessor post-close hooks at 1731236623325 (+3 ms)Closed at 1731236623325 2024-11-10T11:03:43,325 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1731236590709.6dc8a2fd92092c8fc02930629a7fa4ce. 2024-11-10T11:03:43,326 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4969bd98a51c96ecbb473e3472dc6531, disabling compactions & flushes 2024-11-10T11:03:43,326 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:43,326 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:43,326 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. after waiting 0 ms 2024-11-10T11:03:43,326 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:43,326 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed->hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/1e67176538e0c6b983734c0b73e922ed/info/30aab023d5244b8aae8eec9ddb98b730-top, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/eaef03b565984cc694d0396bbaa03553, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/987ec16b1bba4aa29834728bcaef3da7, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a91844cc7eee4c96b5734ee4a6ab0e72, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/ba0683275623423dbd0e5f8357822197, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/83dfb19d93e0423b89737e7ef3be08c0, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/e89117f1e8d24debaaf5ca9d65978a31, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c576d4dab37944ddb8edebe1559cfdfa, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/dc36d066a9d6404595c1a689b49431ea, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/1d6b42e273334fb392a635e16cdc1299, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c32c14948eb9450b9bdf04e2a32a1c45, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7976c6e586514a8aa8ba5a1c9b3ec9de, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a984714cf52b4ac6ba4bfe046d59905c, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a0ce2a50d96049de858364ff8d0428af, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/72e240fd8299464e86286a158510472d, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/75fa1698c12a4213bdef41c2e421c5fd, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/68a2f457bacc489d99a8869adf102ad0, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c8593ccec9844437aeb0cb935344854e, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/bfc0676d76b84b0eb8957f991cdd1372] to archive 2024-11-10T11:03:43,327 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T11:03:43,328 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/30aab023d5244b8aae8eec9ddb98b730.1e67176538e0c6b983734c0b73e922ed 2024-11-10T11:03:43,329 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-3f29ecfe653449d9a0acdb54d47e45f7 2024-11-10T11:03:43,330 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-f11f2fd444a64c27aa5386c6e4de6e3b 2024-11-10T11:03:43,331 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/eaef03b565984cc694d0396bbaa03553 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/eaef03b565984cc694d0396bbaa03553 2024-11-10T11:03:43,332 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/TestLogRolling-testLogRolling=1e67176538e0c6b983734c0b73e922ed-55e866048d3e41a9ab1dfc3ea410e442 2024-11-10T11:03:43,333 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/987ec16b1bba4aa29834728bcaef3da7 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/987ec16b1bba4aa29834728bcaef3da7 2024-11-10T11:03:43,333 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a91844cc7eee4c96b5734ee4a6ab0e72 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a91844cc7eee4c96b5734ee4a6ab0e72 2024-11-10T11:03:43,334 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/ba0683275623423dbd0e5f8357822197 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/ba0683275623423dbd0e5f8357822197 2024-11-10T11:03:43,335 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/83dfb19d93e0423b89737e7ef3be08c0 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/83dfb19d93e0423b89737e7ef3be08c0 2024-11-10T11:03:43,336 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/e89117f1e8d24debaaf5ca9d65978a31 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/e89117f1e8d24debaaf5ca9d65978a31 2024-11-10T11:03:43,337 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c576d4dab37944ddb8edebe1559cfdfa to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c576d4dab37944ddb8edebe1559cfdfa 2024-11-10T11:03:43,338 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/dc36d066a9d6404595c1a689b49431ea to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/dc36d066a9d6404595c1a689b49431ea 2024-11-10T11:03:43,339 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/1d6b42e273334fb392a635e16cdc1299 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/1d6b42e273334fb392a635e16cdc1299 2024-11-10T11:03:43,340 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c32c14948eb9450b9bdf04e2a32a1c45 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c32c14948eb9450b9bdf04e2a32a1c45 2024-11-10T11:03:43,341 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7976c6e586514a8aa8ba5a1c9b3ec9de to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/7976c6e586514a8aa8ba5a1c9b3ec9de 2024-11-10T11:03:43,342 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a984714cf52b4ac6ba4bfe046d59905c to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a984714cf52b4ac6ba4bfe046d59905c 2024-11-10T11:03:43,343 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a0ce2a50d96049de858364ff8d0428af to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/a0ce2a50d96049de858364ff8d0428af 2024-11-10T11:03:43,344 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/72e240fd8299464e86286a158510472d to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/72e240fd8299464e86286a158510472d 2024-11-10T11:03:43,345 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/75fa1698c12a4213bdef41c2e421c5fd to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/75fa1698c12a4213bdef41c2e421c5fd 2024-11-10T11:03:43,346 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/68a2f457bacc489d99a8869adf102ad0 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/68a2f457bacc489d99a8869adf102ad0 2024-11-10T11:03:43,347 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c8593ccec9844437aeb0cb935344854e to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/c8593ccec9844437aeb0cb935344854e 2024-11-10T11:03:43,348 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/bfc0676d76b84b0eb8957f991cdd1372 to hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/archive/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/info/bfc0676d76b84b0eb8957f991cdd1372 2024-11-10T11:03:43,348 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [eaef03b565984cc694d0396bbaa03553=43081, 987ec16b1bba4aa29834728bcaef3da7=12516, a91844cc7eee4c96b5734ee4a6ab0e72=66967, ba0683275623423dbd0e5f8357822197=21156, 83dfb19d93e0423b89737e7ef3be08c0=16828, e89117f1e8d24debaaf5ca9d65978a31=94096, c576d4dab37944ddb8edebe1559cfdfa=20078, dc36d066a9d6404595c1a689b49431ea=20078, 1d6b42e273334fb392a635e16cdc1299=116840, c32c14948eb9450b9bdf04e2a32a1c45=12516, 7976c6e586514a8aa8ba5a1c9b3ec9de=24394, a984714cf52b4ac6ba4bfe046d59905c=148409, a0ce2a50d96049de858364ff8d0428af=16828, 72e240fd8299464e86286a158510472d=22251, 75fa1698c12a4213bdef41c2e421c5fd=180886, 68a2f457bacc489d99a8869adf102ad0=20092, c8593ccec9844437aeb0cb935344854e=12523, bfc0676d76b84b0eb8957f991cdd1372=21171] 2024-11-10T11:03:43,352 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/data/default/TestLogRolling-testLogRolling/4969bd98a51c96ecbb473e3472dc6531/recovered.edits/335.seqid, newMaxSeqId=335, maxSeqId=126 2024-11-10T11:03:43,352 INFO [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:43,352 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4969bd98a51c96ecbb473e3472dc6531: Waiting for close lock at 1731236623326Running coprocessor pre-close hooks at 1731236623326Disabling compacts and flushes for region at 1731236623326Disabling writes for close at 1731236623326Writing region close event to WAL at 1731236623349 (+23 ms)Running coprocessor post-close hooks at 1731236623352 (+3 ms)Closed at 1731236623352 2024-11-10T11:03:43,352 DEBUG [RS_CLOSE_REGION-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1731236590709.4969bd98a51c96ecbb473e3472dc6531. 2024-11-10T11:03:43,518 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,36119,1731236575621; all regions closed. 2024-11-10T11:03:43,519 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,519 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,519 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,519 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,519 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741834_1010 (size=8107) 2024-11-10T11:03:43,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741834_1010 (size=8107) 2024-11-10T11:03:43,523 DEBUG [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/oldWALs 2024-11-10T11:03:43,523 INFO [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C36119%2C1731236575621.meta:.meta(num 1731236576370) 2024-11-10T11:03:43,523 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,523 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,524 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,524 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,524 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741877_1053 (size=780) 2024-11-10T11:03:43,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741877_1053 (size=780) 2024-11-10T11:03:43,527 DEBUG [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/oldWALs 2024-11-10T11:03:43,527 INFO [RS:0;eccb6f743e07:36119 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C36119%2C1731236575621:(num 1731236623205) 2024-11-10T11:03:43,527 DEBUG [RS:0;eccb6f743e07:36119 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:43,527 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:03:43,527 INFO [RS:0;eccb6f743e07:36119 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:03:43,527 INFO [RS:0;eccb6f743e07:36119 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T11:03:43,527 INFO [RS:0;eccb6f743e07:36119 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:03:43,528 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:03:43,528 INFO [RS:0;eccb6f743e07:36119 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36119 2024-11-10T11:03:43,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:03:43,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,36119,1731236575621 2024-11-10T11:03:43,530 INFO [RS:0;eccb6f743e07:36119 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:03:43,531 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,36119,1731236575621] 2024-11-10T11:03:43,533 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,36119,1731236575621 already deleted, retry=false 2024-11-10T11:03:43,533 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,36119,1731236575621 expired; onlineServers=0 2024-11-10T11:03:43,533 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,42865,1731236575570' ***** 2024-11-10T11:03:43,533 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:03:43,533 INFO [M:0;eccb6f743e07:42865 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:03:43,533 INFO [M:0;eccb6f743e07:42865 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:03:43,533 DEBUG [M:0;eccb6f743e07:42865 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:03:43,533 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:03:43,533 DEBUG [M:0;eccb6f743e07:42865 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:03:43,533 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236575768 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236575768,5,FailOnTimeoutGroup] 2024-11-10T11:03:43,533 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236575768 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236575768,5,FailOnTimeoutGroup] 2024-11-10T11:03:43,534 INFO [M:0;eccb6f743e07:42865 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:03:43,534 INFO [M:0;eccb6f743e07:42865 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:03:43,534 DEBUG [M:0;eccb6f743e07:42865 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:03:43,534 INFO [M:0;eccb6f743e07:42865 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:03:43,534 INFO [M:0;eccb6f743e07:42865 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:03:43,534 INFO [M:0;eccb6f743e07:42865 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:03:43,534 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:03:43,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:03:43,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:43,535 DEBUG [M:0;eccb6f743e07:42865 {}] zookeeper.ZKUtil(347): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:03:43,535 WARN [M:0;eccb6f743e07:42865 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:03:43,535 INFO [M:0;eccb6f743e07:42865 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/.lastflushedseqids 2024-11-10T11:03:43,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741878_1054 (size=228) 2024-11-10T11:03:43,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741878_1054 (size=228) 2024-11-10T11:03:43,541 INFO [M:0;eccb6f743e07:42865 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:03:43,541 INFO [M:0;eccb6f743e07:42865 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:03:43,541 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:03:43,541 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:43,541 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:43,541 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:03:43,541 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:43,541 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=53.71 KB heapSize=65.93 KB 2024-11-10T11:03:43,557 DEBUG [M:0;eccb6f743e07:42865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fa29812c050f44dba90dd41bccb9dcc7 is 82, key is hbase:meta,,1/info:regioninfo/1731236576392/Put/seqid=0 2024-11-10T11:03:43,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741879_1055 (size=5672) 2024-11-10T11:03:43,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741879_1055 (size=5672) 2024-11-10T11:03:43,562 INFO [M:0;eccb6f743e07:42865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fa29812c050f44dba90dd41bccb9dcc7 2024-11-10T11:03:43,580 DEBUG [M:0;eccb6f743e07:42865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9be6ef954c249b4a93a4819841cc132 is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731236576816/Put/seqid=0 2024-11-10T11:03:43,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741880_1056 (size=7681) 2024-11-10T11:03:43,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741880_1056 (size=7681) 2024-11-10T11:03:43,586 INFO [M:0;eccb6f743e07:42865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.11 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9be6ef954c249b4a93a4819841cc132 2024-11-10T11:03:43,590 INFO [M:0;eccb6f743e07:42865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f9be6ef954c249b4a93a4819841cc132 2024-11-10T11:03:43,603 DEBUG [M:0;eccb6f743e07:42865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0642af21763b4d7dbf4e48aa620d26bb is 69, key is eccb6f743e07,36119,1731236575621/rs:state/1731236575856/Put/seqid=0 2024-11-10T11:03:43,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741881_1057 (size=5156) 2024-11-10T11:03:43,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741881_1057 (size=5156) 2024-11-10T11:03:43,608 INFO [M:0;eccb6f743e07:42865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0642af21763b4d7dbf4e48aa620d26bb 2024-11-10T11:03:43,626 DEBUG [M:0;eccb6f743e07:42865 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd16d840510749f18c12f23acaf6610d is 52, key is load_balancer_on/state:d/1731236576447/Put/seqid=0 2024-11-10T11:03:43,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741882_1058 (size=5056) 2024-11-10T11:03:43,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741882_1058 (size=5056) 2024-11-10T11:03:43,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:43,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:43,631 INFO [M:0;eccb6f743e07:42865 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd16d840510749f18c12f23acaf6610d 2024-11-10T11:03:43,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:43,631 INFO [RS:0;eccb6f743e07:36119 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:03:43,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36119-0x101018ba3d30001, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:43,631 INFO [RS:0;eccb6f743e07:36119 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,36119,1731236575621; zookeeper connection closed. 2024-11-10T11:03:43,632 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7603e83b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7603e83b 2024-11-10T11:03:43,632 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T11:03:43,637 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fa29812c050f44dba90dd41bccb9dcc7 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fa29812c050f44dba90dd41bccb9dcc7 2024-11-10T11:03:43,641 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fa29812c050f44dba90dd41bccb9dcc7, entries=8, sequenceid=129, filesize=5.5 K 2024-11-10T11:03:43,642 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f9be6ef954c249b4a93a4819841cc132 as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f9be6ef954c249b4a93a4819841cc132 2024-11-10T11:03:43,646 INFO [M:0;eccb6f743e07:42865 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f9be6ef954c249b4a93a4819841cc132 2024-11-10T11:03:43,646 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f9be6ef954c249b4a93a4819841cc132, entries=14, sequenceid=129, filesize=7.5 K 2024-11-10T11:03:43,647 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0642af21763b4d7dbf4e48aa620d26bb as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0642af21763b4d7dbf4e48aa620d26bb 2024-11-10T11:03:43,651 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0642af21763b4d7dbf4e48aa620d26bb, entries=1, sequenceid=129, filesize=5.0 K 2024-11-10T11:03:43,652 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/bd16d840510749f18c12f23acaf6610d as hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bd16d840510749f18c12f23acaf6610d 2024-11-10T11:03:43,656 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:41611/user/jenkins/test-data/41f29fd9-8b1b-4f45-bf2f-515c51e44ef0/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/bd16d840510749f18c12f23acaf6610d, entries=1, sequenceid=129, filesize=4.9 K 2024-11-10T11:03:43,658 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=129, compaction requested=false 2024-11-10T11:03:43,659 INFO [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:43,659 DEBUG [M:0;eccb6f743e07:42865 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236623541Disabling compacts and flushes for region at 1731236623541Disabling writes for close at 1731236623541Obtaining lock to block concurrent updates at 1731236623541Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236623541Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=54997, getHeapSize=67448, getOffHeapSize=0, getCellsCount=152 at 1731236623541Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236623542 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236623542Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236623557 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236623557Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236623566 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236623580 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236623580Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236623590 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236623603 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236623603Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236623612 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236623626 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236623626Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b26b475: reopening flushed file at 1731236623636 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@137a1f42: reopening flushed file at 1731236623641 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ff141c5: reopening flushed file at 1731236623646 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3509d6fb: reopening flushed file at 1731236623651 (+5 ms)Finished flush of dataSize ~53.71 KB/54997, heapSize ~65.87 KB/67448, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 116ms, sequenceid=129, compaction requested=false at 1731236623658 (+7 ms)Writing region close event to WAL at 1731236623659 (+1 ms)Closed at 1731236623659 2024-11-10T11:03:43,659 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,660 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,660 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,660 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,660 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:43,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42175 is added to blk_1073741830_1006 (size=63927) 2024-11-10T11:03:43,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43697 is added to blk_1073741830_1006 (size=63927) 2024-11-10T11:03:43,662 INFO [M:0;eccb6f743e07:42865 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:03:43,662 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:03:43,662 INFO [M:0;eccb6f743e07:42865 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42865 2024-11-10T11:03:43,663 INFO [M:0;eccb6f743e07:42865 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:03:43,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:43,765 INFO [M:0;eccb6f743e07:42865 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:03:43,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42865-0x101018ba3d30000, quorum=127.0.0.1:59422, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:43,767 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@76385d85{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:03:43,767 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@284a8a01{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:03:43,767 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:03:43,768 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7193a060{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:03:43,768 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@444d0b71{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir/,STOPPED} 2024-11-10T11:03:43,770 WARN [BP-1385195385-172.17.0.2-1731236574859 heartbeating to localhost/127.0.0.1:41611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:03:43,770 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:03:43,770 WARN [BP-1385195385-172.17.0.2-1731236574859 heartbeating to localhost/127.0.0.1:41611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1385195385-172.17.0.2-1731236574859 (Datanode Uuid 8f52354b-388b-417a-b058-9ce882a6e6d6) service to localhost/127.0.0.1:41611 2024-11-10T11:03:43,770 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:03:43,770 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data3/current/BP-1385195385-172.17.0.2-1731236574859 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:43,770 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data4/current/BP-1385195385-172.17.0.2-1731236574859 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:43,771 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:03:43,772 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27e7702e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:03:43,773 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@33382c80{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:03:43,773 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:03:43,773 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3c10c155{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:03:43,773 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a83b9f6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir/,STOPPED} 2024-11-10T11:03:43,775 WARN [BP-1385195385-172.17.0.2-1731236574859 heartbeating to localhost/127.0.0.1:41611 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:03:43,775 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:03:43,775 WARN [BP-1385195385-172.17.0.2-1731236574859 heartbeating to localhost/127.0.0.1:41611 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1385195385-172.17.0.2-1731236574859 (Datanode Uuid 886933b3-9da5-4304-9f40-94ffebbc8dfc) service to localhost/127.0.0.1:41611 2024-11-10T11:03:43,775 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:03:43,775 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data1/current/BP-1385195385-172.17.0.2-1731236574859 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:43,775 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/cluster_93349c44-1ab6-8b65-dcaa-147f13bce2ac/data/data2/current/BP-1385195385-172.17.0.2-1731236574859 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:43,776 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:03:43,781 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2ffc76d1{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:03:43,782 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@640c9e09{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:03:43,782 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:03:43,782 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7e466bdb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:03:43,782 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3d320e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir/,STOPPED} 2024-11-10T11:03:43,789 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:03:43,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:03:43,824 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=232 (was 205) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41611 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:41611 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41611 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41611 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41611 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41611 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41611 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41611 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:41611 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:41611 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=506 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=23 (was 39), ProcessCount=11 (was 11), AvailableMemoryMB=8604 (was 8649) 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=232, OpenFileDescriptor=506, MaxFileDescriptor=1048576, SystemLoadAverage=23, ProcessCount=11, AvailableMemoryMB=8605 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.log.dir so I do NOT create it in target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e7769794-76aa-2346-72cc-a678ab94564c/hadoop.tmp.dir so I do NOT create it in target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7, deleteOnExit=true 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/test.cache.data in system properties and HBase conf 2024-11-10T11:03:43,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T11:03:43,834 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/nfs.dump.dir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/java.io.tmpdir in system properties and HBase conf 2024-11-10T11:03:43,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T11:03:43,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T11:03:43,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T11:03:43,848 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:03:43,869 INFO [regionserver/eccb6f743e07:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:03:43,903 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:03:43,907 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:03:43,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:03:43,916 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:03:43,916 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:03:43,917 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:03:43,917 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@46dfb375{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:03:43,918 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2a7c9e07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:03:44,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30964c14{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/java.io.tmpdir/jetty-localhost-36801-hadoop-hdfs-3_4_1-tests_jar-_-any-3455773200051853127/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:03:44,032 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@32d01bcd{HTTP/1.1, (http/1.1)}{localhost:36801} 2024-11-10T11:03:44,032 INFO [Time-limited test {}] server.Server(415): Started @284603ms 2024-11-10T11:03:44,044 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-10T11:03:44,095 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:03:44,097 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:03:44,098 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:03:44,098 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:03:44,098 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:03:44,099 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d72fbcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:03:44,099 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1bfccdd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:03:44,213 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@186dc28{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/java.io.tmpdir/jetty-localhost-35141-hadoop-hdfs-3_4_1-tests_jar-_-any-14435310043610249730/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:03:44,214 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c8a9439{HTTP/1.1, (http/1.1)}{localhost:35141} 2024-11-10T11:03:44,214 INFO [Time-limited test {}] server.Server(415): Started @284785ms 2024-11-10T11:03:44,215 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:03:44,243 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T11:03:44,245 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T11:03:44,246 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T11:03:44,246 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T11:03:44,246 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T11:03:44,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7a47ab0e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir/,AVAILABLE} 2024-11-10T11:03:44,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@735ef7ff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T11:03:44,316 WARN [Thread-2465 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data1/current/BP-1022027770-172.17.0.2-1731236623853/current, will proceed with Du for space computation calculation, 2024-11-10T11:03:44,316 WARN [Thread-2466 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data2/current/BP-1022027770-172.17.0.2-1731236623853/current, will proceed with Du for space computation calculation, 2024-11-10T11:03:44,332 WARN [Thread-2444 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:03:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe98df78ba0b8f3f1 with lease ID 0xc57eb97cd1232c29: Processing first storage report for DS-a46d5ae1-e883-4471-b218-611bc195d1a1 from datanode DatanodeRegistration(127.0.0.1:46667, datanodeUuid=f7d019af-6efb-4f36-95ff-15bf0fb9f7a0, infoPort=38281, infoSecurePort=0, ipcPort=39127, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853) 2024-11-10T11:03:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe98df78ba0b8f3f1 with lease ID 0xc57eb97cd1232c29: from storage DS-a46d5ae1-e883-4471-b218-611bc195d1a1 node DatanodeRegistration(127.0.0.1:46667, datanodeUuid=f7d019af-6efb-4f36-95ff-15bf0fb9f7a0, infoPort=38281, infoSecurePort=0, ipcPort=39127, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:03:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe98df78ba0b8f3f1 with lease ID 0xc57eb97cd1232c29: Processing first storage report for DS-3aa7ebf2-0ab0-47fb-8fc1-ce4c4227f98a from datanode DatanodeRegistration(127.0.0.1:46667, datanodeUuid=f7d019af-6efb-4f36-95ff-15bf0fb9f7a0, infoPort=38281, infoSecurePort=0, ipcPort=39127, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853) 2024-11-10T11:03:44,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe98df78ba0b8f3f1 with lease ID 0xc57eb97cd1232c29: from storage DS-3aa7ebf2-0ab0-47fb-8fc1-ce4c4227f98a node DatanodeRegistration(127.0.0.1:46667, datanodeUuid=f7d019af-6efb-4f36-95ff-15bf0fb9f7a0, infoPort=38281, infoSecurePort=0, ipcPort=39127, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:03:44,376 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@10e33487{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/java.io.tmpdir/jetty-localhost-38803-hadoop-hdfs-3_4_1-tests_jar-_-any-6532202008265251810/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:03:44,376 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@814e400{HTTP/1.1, (http/1.1)}{localhost:38803} 2024-11-10T11:03:44,376 INFO [Time-limited test {}] server.Server(415): Started @284947ms 2024-11-10T11:03:44,377 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T11:03:44,467 WARN [Thread-2492 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data4/current/BP-1022027770-172.17.0.2-1731236623853/current, will proceed with Du for space computation calculation, 2024-11-10T11:03:44,468 WARN [Thread-2491 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data3/current/BP-1022027770-172.17.0.2-1731236623853/current, will proceed with Du for space computation calculation, 2024-11-10T11:03:44,489 WARN [Thread-2480 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T11:03:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7323f41872d698a with lease ID 0xc57eb97cd1232c2a: Processing first storage report for DS-b1cebed4-82ff-4ea0-9d47-d7426994d872 from datanode DatanodeRegistration(127.0.0.1:40079, datanodeUuid=82b27ece-91e3-480c-aa28-1ae66fb1ad98, infoPort=36637, infoSecurePort=0, ipcPort=43807, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853) 2024-11-10T11:03:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7323f41872d698a with lease ID 0xc57eb97cd1232c2a: from storage DS-b1cebed4-82ff-4ea0-9d47-d7426994d872 node DatanodeRegistration(127.0.0.1:40079, datanodeUuid=82b27ece-91e3-480c-aa28-1ae66fb1ad98, infoPort=36637, infoSecurePort=0, ipcPort=43807, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:03:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7323f41872d698a with lease ID 0xc57eb97cd1232c2a: Processing first storage report for DS-d3a3d55d-4e7b-456a-a75a-1e0babf2949e from datanode DatanodeRegistration(127.0.0.1:40079, datanodeUuid=82b27ece-91e3-480c-aa28-1ae66fb1ad98, infoPort=36637, infoSecurePort=0, ipcPort=43807, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853) 2024-11-10T11:03:44,491 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7323f41872d698a with lease ID 0xc57eb97cd1232c2a: from storage DS-d3a3d55d-4e7b-456a-a75a-1e0babf2949e node DatanodeRegistration(127.0.0.1:40079, datanodeUuid=82b27ece-91e3-480c-aa28-1ae66fb1ad98, infoPort=36637, infoSecurePort=0, ipcPort=43807, storageInfo=lv=-57;cid=testClusterID;nsid=1643349293;c=1731236623853), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T11:03:44,498 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04 2024-11-10T11:03:44,500 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/zookeeper_0, clientPort=62880, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T11:03:44,501 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62880 2024-11-10T11:03:44,501 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:03:44,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741825_1001 (size=7) 2024-11-10T11:03:44,512 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b with version=8 2024-11-10T11:03:44,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:33151/user/jenkins/test-data/c815a979-7854-dc08-8b92-f8e7199dee1e/hbase-staging 2024-11-10T11:03:44,514 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T11:03:44,514 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:03:44,515 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45435 2024-11-10T11:03:44,516 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45435 connecting to ZooKeeper ensemble=127.0.0.1:62880 2024-11-10T11:03:44,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:454350x0, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:03:44,523 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45435-0x101018c63030000 connected 2024-11-10T11:03:44,540 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,542 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,544 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:03:44,544 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b, hbase.cluster.distributed=false 2024-11-10T11:03:44,545 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:03:44,546 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45435 2024-11-10T11:03:44,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45435 2024-11-10T11:03:44,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45435 2024-11-10T11:03:44,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45435 2024-11-10T11:03:44,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45435 2024-11-10T11:03:44,564 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eccb6f743e07:0 server-side Connection retries=45 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T11:03:44,564 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T11:03:44,565 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40985 2024-11-10T11:03:44,566 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40985 connecting to ZooKeeper ensemble=127.0.0.1:62880 2024-11-10T11:03:44,566 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,568 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409850x0, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T11:03:44,573 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:409850x0, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:03:44,573 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40985-0x101018c63030001 connected 2024-11-10T11:03:44,573 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T11:03:44,574 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T11:03:44,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T11:03:44,575 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T11:03:44,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40985 2024-11-10T11:03:44,575 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40985 2024-11-10T11:03:44,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40985 2024-11-10T11:03:44,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40985 2024-11-10T11:03:44,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40985 2024-11-10T11:03:44,587 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eccb6f743e07:45435 2024-11-10T11:03:44,588 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:03:44,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:03:44,589 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T11:03:44,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,591 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T11:03:44,591 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eccb6f743e07,45435,1731236624514 from backup master directory 2024-11-10T11:03:44,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:03:44,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T11:03:44,593 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:03:44,593 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,596 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/hbase.id] with ID: b47dcfe1-c998-47b3-8d3b-5968e4fe9fa5 2024-11-10T11:03:44,596 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/.tmp/hbase.id 2024-11-10T11:03:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:03:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741826_1002 (size=42) 2024-11-10T11:03:44,601 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/.tmp/hbase.id]:[hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/hbase.id] 2024-11-10T11:03:44,612 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:44,612 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T11:03:44,613 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T11:03:44,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:03:44,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741827_1003 (size=196) 2024-11-10T11:03:44,622 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T11:03:44,623 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T11:03:44,623 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:03:44,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:03:44,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741828_1004 (size=1189) 2024-11-10T11:03:44,631 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store 2024-11-10T11:03:44,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:44,631 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:03:44,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741829_1005 (size=34) 2024-11-10T11:03:44,637 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:03:44,637 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:03:44,637 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:44,637 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:44,637 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:03:44,637 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:44,637 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:44,637 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236624637Disabling compacts and flushes for region at 1731236624637Disabling writes for close at 1731236624637Writing region close event to WAL at 1731236624637Closed at 1731236624637 2024-11-10T11:03:44,638 WARN [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/.initializing 2024-11-10T11:03:44,638 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/WALs/eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,641 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C45435%2C1731236624514, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/WALs/eccb6f743e07,45435,1731236624514, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/oldWALs, maxLogs=10 2024-11-10T11:03:44,641 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C45435%2C1731236624514.1731236624641 2024-11-10T11:03:44,646 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/WALs/eccb6f743e07,45435,1731236624514/eccb6f743e07%2C45435%2C1731236624514.1731236624641 2024-11-10T11:03:44,650 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38281:38281),(127.0.0.1/127.0.0.1:36637:36637)] 2024-11-10T11:03:44,652 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:03:44,652 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:03:44,653 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,653 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T11:03:44,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:44,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T11:03:44,657 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:03:44,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T11:03:44,658 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:03:44,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T11:03:44,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T11:03:44,661 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,661 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,661 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,662 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,663 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,663 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T11:03:44,664 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T11:03:44,668 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:03:44,669 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=749609, jitterRate=-0.046823650598526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T11:03:44,669 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731236624653Initializing all the Stores at 1731236624654 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236624654Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236624654Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236624654Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236624654Cleaning up temporary data from old regions at 1731236624663 (+9 ms)Region opened successfully at 1731236624669 (+6 ms) 2024-11-10T11:03:44,672 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T11:03:44,675 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9399cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:03:44,676 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T11:03:44,676 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T11:03:44,676 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T11:03:44,676 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T11:03:44,676 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T11:03:44,677 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T11:03:44,677 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T11:03:44,679 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T11:03:44,679 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T11:03:44,681 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T11:03:44,681 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T11:03:44,682 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T11:03:44,683 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T11:03:44,683 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T11:03:44,684 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T11:03:44,685 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T11:03:44,686 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T11:03:44,688 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T11:03:44,689 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T11:03:44,691 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T11:03:44,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:03:44,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T11:03:44,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,693 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eccb6f743e07,45435,1731236624514, sessionid=0x101018c63030000, setting cluster-up flag (Was=false) 2024-11-10T11:03:44,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,701 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T11:03:44,702 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:44,711 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T11:03:44,712 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eccb6f743e07,45435,1731236624514 2024-11-10T11:03:44,713 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T11:03:44,715 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T11:03:44,715 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T11:03:44,715 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T11:03:44,716 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eccb6f743e07,45435,1731236624514 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eccb6f743e07:0, corePoolSize=5, maxPoolSize=5 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eccb6f743e07:0, corePoolSize=10, maxPoolSize=10 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:03:44,717 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,718 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731236654718 2024-11-10T11:03:44,718 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T11:03:44,719 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:03:44,719 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T11:03:44,719 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T11:03:44,720 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T11:03:44,720 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T11:03:44,720 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,720 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236624720,5,FailOnTimeoutGroup] 2024-11-10T11:03:44,720 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T11:03:44,724 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236624720,5,FailOnTimeoutGroup] 2024-11-10T11:03:44,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T11:03:44,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,724 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:03:44,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741831_1007 (size=1321) 2024-11-10T11:03:44,731 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T11:03:44,731 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b 2024-11-10T11:03:44,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:03:44,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741832_1008 (size=32) 2024-11-10T11:03:44,741 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:03:44,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:03:44,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:03:44,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:44,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:03:44,747 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:03:44,747 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:44,747 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:03:44,748 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:03:44,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:44,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:03:44,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:03:44,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:44,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:44,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:03:44,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740 2024-11-10T11:03:44,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740 2024-11-10T11:03:44,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:03:44,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:03:44,753 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:03:44,754 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:03:44,756 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T11:03:44,756 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=841240, jitterRate=0.0696922242641449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:03:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731236624741Initializing all the Stores at 1731236624742 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236624742Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236624744 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236624744Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236624744Cleaning up temporary data from old regions at 1731236624753 (+9 ms)Region opened successfully at 1731236624757 (+4 ms) 2024-11-10T11:03:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:03:44,757 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:03:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:03:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:03:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:03:44,757 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:03:44,757 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236624757Disabling compacts and flushes for region at 1731236624757Disabling writes for close at 1731236624757Writing region close event to WAL at 1731236624757Closed at 1731236624757 2024-11-10T11:03:44,758 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:03:44,758 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T11:03:44,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T11:03:44,760 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:03:44,760 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T11:03:44,778 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(746): ClusterId : b47dcfe1-c998-47b3-8d3b-5968e4fe9fa5 2024-11-10T11:03:44,778 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T11:03:44,780 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T11:03:44,780 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T11:03:44,782 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T11:03:44,782 DEBUG [RS:0;eccb6f743e07:40985 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e78882f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eccb6f743e07/172.17.0.2:0 2024-11-10T11:03:44,794 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eccb6f743e07:40985 2024-11-10T11:03:44,794 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T11:03:44,794 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T11:03:44,794 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T11:03:44,795 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(2659): reportForDuty to master=eccb6f743e07,45435,1731236624514 with port=40985, startcode=1731236624564 2024-11-10T11:03:44,795 DEBUG [RS:0;eccb6f743e07:40985 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T11:03:44,797 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37943, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T11:03:44,798 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45435 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eccb6f743e07,40985,1731236624564 2024-11-10T11:03:44,798 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45435 {}] master.ServerManager(517): Registering regionserver=eccb6f743e07,40985,1731236624564 2024-11-10T11:03:44,799 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b 2024-11-10T11:03:44,799 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44203 2024-11-10T11:03:44,799 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T11:03:44,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:03:44,801 DEBUG [RS:0;eccb6f743e07:40985 {}] zookeeper.ZKUtil(111): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eccb6f743e07,40985,1731236624564 2024-11-10T11:03:44,801 WARN [RS:0;eccb6f743e07:40985 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T11:03:44,801 INFO [RS:0;eccb6f743e07:40985 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:03:44,801 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/eccb6f743e07,40985,1731236624564 2024-11-10T11:03:44,801 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eccb6f743e07,40985,1731236624564] 2024-11-10T11:03:44,804 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T11:03:44,805 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T11:03:44,805 INFO [RS:0;eccb6f743e07:40985 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T11:03:44,805 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,806 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T11:03:44,806 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T11:03:44,806 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,806 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,806 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,806 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,806 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,806 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eccb6f743e07:0, corePoolSize=2, maxPoolSize=2 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eccb6f743e07:0, corePoolSize=1, maxPoolSize=1 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:03:44,807 DEBUG [RS:0;eccb6f743e07:40985 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eccb6f743e07:0, corePoolSize=3, maxPoolSize=3 2024-11-10T11:03:44,807 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,807 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,807 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,807 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,807 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,807 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40985,1731236624564-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:03:44,821 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T11:03:44,822 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,40985,1731236624564-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,822 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,822 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.Replication(171): eccb6f743e07,40985,1731236624564 started 2024-11-10T11:03:44,835 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:44,836 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1482): Serving as eccb6f743e07,40985,1731236624564, RpcServer on eccb6f743e07/172.17.0.2:40985, sessionid=0x101018c63030001 2024-11-10T11:03:44,836 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T11:03:44,836 DEBUG [RS:0;eccb6f743e07:40985 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eccb6f743e07,40985,1731236624564 2024-11-10T11:03:44,836 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,40985,1731236624564' 2024-11-10T11:03:44,836 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T11:03:44,836 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eccb6f743e07,40985,1731236624564 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eccb6f743e07,40985,1731236624564' 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T11:03:44,837 DEBUG [RS:0;eccb6f743e07:40985 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T11:03:44,837 INFO [RS:0;eccb6f743e07:40985 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T11:03:44,837 INFO [RS:0;eccb6f743e07:40985 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T11:03:44,911 WARN [eccb6f743e07:45435 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T11:03:44,939 INFO [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C40985%2C1731236624564, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/eccb6f743e07,40985,1731236624564, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs, maxLogs=32 2024-11-10T11:03:44,940 INFO [RS:0;eccb6f743e07:40985 {}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C40985%2C1731236624564.1731236624940 2024-11-10T11:03:44,945 INFO [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/eccb6f743e07,40985,1731236624564/eccb6f743e07%2C40985%2C1731236624564.1731236624940 2024-11-10T11:03:44,946 DEBUG [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36637:36637),(127.0.0.1/127.0.0.1:38281:38281)] 2024-11-10T11:03:44,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:03:44,968 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T11:03:44,969 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-10T11:03:45,161 DEBUG [eccb6f743e07:45435 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T11:03:45,161 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eccb6f743e07,40985,1731236624564 2024-11-10T11:03:45,162 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,40985,1731236624564, state=OPENING 2024-11-10T11:03:45,165 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T11:03:45,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:45,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:45,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:03:45,167 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T11:03:45,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:03:45,167 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,40985,1731236624564}] 2024-11-10T11:03:45,320 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T11:03:45,322 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T11:03:45,325 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T11:03:45,325 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:03:45,326 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eccb6f743e07%2C40985%2C1731236624564.meta, suffix=.meta, logDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/eccb6f743e07,40985,1731236624564, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs, maxLogs=32 2024-11-10T11:03:45,327 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor eccb6f743e07%2C40985%2C1731236624564.meta.1731236625327.meta 2024-11-10T11:03:45,333 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/eccb6f743e07,40985,1731236624564/eccb6f743e07%2C40985%2C1731236624564.meta.1731236625327.meta 2024-11-10T11:03:45,334 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38281:38281),(127.0.0.1/127.0.0.1:36637:36637)] 2024-11-10T11:03:45,340 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T11:03:45,341 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T11:03:45,341 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T11:03:45,341 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T11:03:45,341 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T11:03:45,341 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T11:03:45,341 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T11:03:45,341 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T11:03:45,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T11:03:45,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T11:03:45,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:45,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:45,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T11:03:45,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T11:03:45,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:45,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:45,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T11:03:45,346 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T11:03:45,346 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:45,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:45,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T11:03:45,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T11:03:45,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T11:03:45,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T11:03:45,347 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T11:03:45,348 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740 2024-11-10T11:03:45,349 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740 2024-11-10T11:03:45,350 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T11:03:45,350 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T11:03:45,350 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T11:03:45,351 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T11:03:45,352 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=759456, jitterRate=-0.03430287539958954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T11:03:45,352 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T11:03:45,353 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731236625341Writing region info on filesystem at 1731236625342 (+1 ms)Initializing all the Stores at 1731236625342Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236625342Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236625342Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731236625342Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731236625342Cleaning up temporary data from old regions at 1731236625350 (+8 ms)Running coprocessor post-open hooks at 1731236625352 (+2 ms)Region opened successfully at 1731236625353 (+1 ms) 2024-11-10T11:03:45,354 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731236625319 2024-11-10T11:03:45,355 DEBUG [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T11:03:45,355 INFO [RS_OPEN_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T11:03:45,356 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eccb6f743e07,40985,1731236624564 2024-11-10T11:03:45,357 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eccb6f743e07,40985,1731236624564, state=OPEN 2024-11-10T11:03:45,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:03:45,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T11:03:45,361 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eccb6f743e07,40985,1731236624564 2024-11-10T11:03:45,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:03:45,362 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T11:03:45,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T11:03:45,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eccb6f743e07,40985,1731236624564 in 195 msec 2024-11-10T11:03:45,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T11:03:45,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec 2024-11-10T11:03:45,366 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T11:03:45,366 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T11:03:45,367 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:03:45,367 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,40985,1731236624564, seqNum=-1] 2024-11-10T11:03:45,368 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:03:45,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56785, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:03:45,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 657 msec 2024-11-10T11:03:45,373 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731236625373, completionTime=-1 2024-11-10T11:03:45,373 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T11:03:45,373 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T11:03:45,374 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T11:03:45,374 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731236685374 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731236745375 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 1 msec 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45435,1731236624514-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45435,1731236624514-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45435,1731236624514-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eccb6f743e07:45435, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,375 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,376 DEBUG [master/eccb6f743e07:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.785sec 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45435,1731236624514-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T11:03:45,378 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45435,1731236624514-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T11:03:45,380 DEBUG [master/eccb6f743e07:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T11:03:45,380 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T11:03:45,380 INFO [master/eccb6f743e07:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eccb6f743e07,45435,1731236624514-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T11:03:45,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3829c39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:03:45,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eccb6f743e07,45435,-1 for getting cluster id 2024-11-10T11:03:45,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T11:03:45,480 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b47dcfe1-c998-47b3-8d3b-5968e4fe9fa5' 2024-11-10T11:03:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T11:03:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b47dcfe1-c998-47b3-8d3b-5968e4fe9fa5" 2024-11-10T11:03:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ff46454, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:03:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eccb6f743e07,45435,-1] 2024-11-10T11:03:45,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T11:03:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:45,482 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44588, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T11:03:45,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64440bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T11:03:45,483 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T11:03:45,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eccb6f743e07,40985,1731236624564, seqNum=-1] 2024-11-10T11:03:45,485 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T11:03:45,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T11:03:45,487 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eccb6f743e07,45435,1731236624514 2024-11-10T11:03:45,487 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T11:03:45,489 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-10T11:03:45,489 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-10T11:03:45,491 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/test.com,8080,1, archiveDir=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs, maxLogs=32 2024-11-10T11:03:45,492 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731236625492 2024-11-10T11:03:45,496 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/test.com,8080,1/test.com%2C8080%2C1.1731236625492 2024-11-10T11:03:45,499 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36637:36637),(127.0.0.1/127.0.0.1:38281:38281)] 2024-11-10T11:03:45,502 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1731236625502 2024-11-10T11:03:45,507 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,507 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,507 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,507 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,507 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,507 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/test.com,8080,1/test.com%2C8080%2C1.1731236625492 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/test.com,8080,1/test.com%2C8080%2C1.1731236625502 2024-11-10T11:03:45,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36637:36637),(127.0.0.1/127.0.0.1:38281:38281)] 2024-11-10T11:03:45,508 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/test.com,8080,1/test.com%2C8080%2C1.1731236625492 is not closed yet, will try archiving it next time 2024-11-10T11:03:45,508 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,508 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741835_1011 (size=93) 2024-11-10T11:03:45,509 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,509 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741835_1011 (size=93) 2024-11-10T11:03:45,509 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,512 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/WALs/test.com,8080,1/test.com%2C8080%2C1.1731236625492 to hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs/test.com%2C8080%2C1.1731236625492 2024-11-10T11:03:45,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741836_1012 (size=93) 2024-11-10T11:03:45,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741836_1012 (size=93) 2024-11-10T11:03:45,515 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs 2024-11-10T11:03:45,515 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1731236625502) 2024-11-10T11:03:45,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T11:03:45,516 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:03:45,516 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:03:45,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:45,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:45,516 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T11:03:45,516 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T11:03:45,516 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1592238995, stopped=false 2024-11-10T11:03:45,516 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eccb6f743e07,45435,1731236624514 2024-11-10T11:03:45,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:03:45,518 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T11:03:45,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:45,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:45,519 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:03:45,519 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T11:03:45,519 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:03:45,519 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:45,519 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eccb6f743e07,40985,1731236624564' ***** 2024-11-10T11:03:45,519 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T11:03:45,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:03:45,519 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T11:03:45,519 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T11:03:45,520 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(959): stopping server eccb6f743e07,40985,1731236624564 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eccb6f743e07:40985. 2024-11-10T11:03:45,520 DEBUG [RS:0;eccb6f743e07:40985 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T11:03:45,520 DEBUG [RS:0;eccb6f743e07:40985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T11:03:45,520 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T11:03:45,520 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T11:03:45,520 DEBUG [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T11:03:45,521 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T11:03:45,521 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T11:03:45,521 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T11:03:45,521 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T11:03:45,521 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T11:03:45,521 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-10T11:03:45,536 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/.tmp/ns/5f8f133961174c258493b1a0345a6590 is 43, key is default/ns:d/1731236625369/Put/seqid=0 2024-11-10T11:03:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741837_1013 (size=5153) 2024-11-10T11:03:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741837_1013 (size=5153) 2024-11-10T11:03:45,541 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/.tmp/ns/5f8f133961174c258493b1a0345a6590 2024-11-10T11:03:45,545 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/.tmp/ns/5f8f133961174c258493b1a0345a6590 as hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/ns/5f8f133961174c258493b1a0345a6590 2024-11-10T11:03:45,549 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/ns/5f8f133961174c258493b1a0345a6590, entries=2, sequenceid=6, filesize=5.0 K 2024-11-10T11:03:45,550 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=6, compaction requested=false 2024-11-10T11:03:45,550 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T11:03:45,554 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T11:03:45,554 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T11:03:45,554 INFO [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T11:03:45,554 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731236625521Running coprocessor pre-close hooks at 1731236625521Disabling compacts and flushes for region at 1731236625521Disabling writes for close at 1731236625521Obtaining lock to block concurrent updates at 1731236625521Preparing flush snapshotting stores in 1588230740 at 1731236625521Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1731236625521Flushing stores of hbase:meta,,1.1588230740 at 1731236625522 (+1 ms)Flushing 1588230740/ns: creating writer at 1731236625522Flushing 1588230740/ns: appending metadata at 1731236625536 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731236625536Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dd21204: reopening flushed file at 1731236625545 (+9 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=6, compaction requested=false at 1731236625550 (+5 ms)Writing region close event to WAL at 1731236625551 (+1 ms)Running coprocessor post-close hooks at 1731236625554 (+3 ms)Closed at 1731236625554 2024-11-10T11:03:45,554 DEBUG [RS_CLOSE_META-regionserver/eccb6f743e07:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T11:03:45,632 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,46705,1731236443672/eccb6f743e07%2C46705%2C1731236443672.meta.1731236444525.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:45,632 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:39141/user/jenkins/test-data/584dd645-c70f-51e5-9b11-009ef1b18f40/WALs/eccb6f743e07,41543,1731236444637/eccb6f743e07%2C41543%2C1731236444637.1731236444825 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-10T11:03:45,721 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(976): stopping server eccb6f743e07,40985,1731236624564; all regions closed. 2024-11-10T11:03:45,721 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,721 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,721 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,721 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,722 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741834_1010 (size=1152) 2024-11-10T11:03:45,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741834_1010 (size=1152) 2024-11-10T11:03:45,725 DEBUG [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs 2024-11-10T11:03:45,726 INFO [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C40985%2C1731236624564.meta:.meta(num 1731236625327) 2024-11-10T11:03:45,726 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,726 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,726 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,726 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,726 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741833_1009 (size=93) 2024-11-10T11:03:45,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741833_1009 (size=93) 2024-11-10T11:03:45,730 DEBUG [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/oldWALs 2024-11-10T11:03:45,730 INFO [RS:0;eccb6f743e07:40985 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog eccb6f743e07%2C40985%2C1731236624564:(num 1731236624940) 2024-11-10T11:03:45,730 DEBUG [RS:0;eccb6f743e07:40985 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T11:03:45,730 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T11:03:45,730 INFO [RS:0;eccb6f743e07:40985 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:03:45,730 INFO [RS:0;eccb6f743e07:40985 {}] hbase.ChoreService(370): Chore service for: regionserver/eccb6f743e07:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T11:03:45,730 INFO [RS:0;eccb6f743e07:40985 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:03:45,730 INFO [regionserver/eccb6f743e07:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:03:45,730 INFO [RS:0;eccb6f743e07:40985 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40985 2024-11-10T11:03:45,732 INFO [RS:0;eccb6f743e07:40985 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:03:45,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T11:03:45,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eccb6f743e07,40985,1731236624564 2024-11-10T11:03:45,734 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eccb6f743e07,40985,1731236624564] 2024-11-10T11:03:45,735 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eccb6f743e07,40985,1731236624564 already deleted, retry=false 2024-11-10T11:03:45,735 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eccb6f743e07,40985,1731236624564 expired; onlineServers=0 2024-11-10T11:03:45,735 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eccb6f743e07,45435,1731236624514' ***** 2024-11-10T11:03:45,735 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T11:03:45,735 INFO [M:0;eccb6f743e07:45435 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T11:03:45,735 INFO [M:0;eccb6f743e07:45435 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T11:03:45,735 DEBUG [M:0;eccb6f743e07:45435 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T11:03:45,735 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T11:03:45,735 DEBUG [M:0;eccb6f743e07:45435 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T11:03:45,735 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236624720 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.small.0-1731236624720,5,FailOnTimeoutGroup] 2024-11-10T11:03:45,735 DEBUG [master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236624720 {}] cleaner.HFileCleaner(306): Exit Thread[master/eccb6f743e07:0:becomeActiveMaster-HFileCleaner.large.0-1731236624720,5,FailOnTimeoutGroup] 2024-11-10T11:03:45,735 INFO [M:0;eccb6f743e07:45435 {}] hbase.ChoreService(370): Chore service for: master/eccb6f743e07:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T11:03:45,735 INFO [M:0;eccb6f743e07:45435 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T11:03:45,736 DEBUG [M:0;eccb6f743e07:45435 {}] master.HMaster(1795): Stopping service threads 2024-11-10T11:03:45,736 INFO [M:0;eccb6f743e07:45435 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T11:03:45,736 INFO [M:0;eccb6f743e07:45435 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T11:03:45,736 INFO [M:0;eccb6f743e07:45435 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T11:03:45,736 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T11:03:45,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T11:03:45,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T11:03:45,738 DEBUG [M:0;eccb6f743e07:45435 {}] zookeeper.ZKUtil(347): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T11:03:45,738 WARN [M:0;eccb6f743e07:45435 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T11:03:45,738 INFO [M:0;eccb6f743e07:45435 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/.lastflushedseqids 2024-11-10T11:03:45,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741838_1014 (size=99) 2024-11-10T11:03:45,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741838_1014 (size=99) 2024-11-10T11:03:45,744 INFO [M:0;eccb6f743e07:45435 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T11:03:45,744 INFO [M:0;eccb6f743e07:45435 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T11:03:45,744 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T11:03:45,744 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:45,744 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:45,744 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T11:03:45,744 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:45,744 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-10T11:03:45,761 DEBUG [M:0;eccb6f743e07:45435 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/36d85b37a24e418180300cec7620275e is 82, key is hbase:meta,,1/info:regioninfo/1731236625356/Put/seqid=0 2024-11-10T11:03:45,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741839_1015 (size=5672) 2024-11-10T11:03:45,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741839_1015 (size=5672) 2024-11-10T11:03:45,765 INFO [M:0;eccb6f743e07:45435 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/36d85b37a24e418180300cec7620275e 2024-11-10T11:03:45,785 DEBUG [M:0;eccb6f743e07:45435 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ee1f0c896d04248be9a5dba280976fe is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1731236625372/Put/seqid=0 2024-11-10T11:03:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741840_1016 (size=5275) 2024-11-10T11:03:45,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741840_1016 (size=5275) 2024-11-10T11:03:45,790 INFO [M:0;eccb6f743e07:45435 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ee1f0c896d04248be9a5dba280976fe 2024-11-10T11:03:45,807 DEBUG [M:0;eccb6f743e07:45435 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e8c7241d3f64eb9b913f4c29d5167c4 is 69, key is eccb6f743e07,40985,1731236624564/rs:state/1731236624798/Put/seqid=0 2024-11-10T11:03:45,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741841_1017 (size=5156) 2024-11-10T11:03:45,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741841_1017 (size=5156) 2024-11-10T11:03:45,812 INFO [M:0;eccb6f743e07:45435 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e8c7241d3f64eb9b913f4c29d5167c4 2024-11-10T11:03:45,829 DEBUG [M:0;eccb6f743e07:45435 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4a74d21257e14f5da7dfd0e16e7ac817 is 52, key is load_balancer_on/state:d/1731236625489/Put/seqid=0 2024-11-10T11:03:45,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741842_1018 (size=5056) 2024-11-10T11:03:45,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741842_1018 (size=5056) 2024-11-10T11:03:45,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:45,834 INFO [RS:0;eccb6f743e07:40985 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:03:45,834 INFO [M:0;eccb6f743e07:45435 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4a74d21257e14f5da7dfd0e16e7ac817 2024-11-10T11:03:45,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40985-0x101018c63030001, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:45,834 INFO [RS:0;eccb6f743e07:40985 {}] regionserver.HRegionServer(1031): Exiting; stopping=eccb6f743e07,40985,1731236624564; zookeeper connection closed. 2024-11-10T11:03:45,834 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d0b256f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d0b256f 2024-11-10T11:03:45,835 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T11:03:45,838 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/36d85b37a24e418180300cec7620275e as hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/36d85b37a24e418180300cec7620275e 2024-11-10T11:03:45,842 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/36d85b37a24e418180300cec7620275e, entries=8, sequenceid=29, filesize=5.5 K 2024-11-10T11:03:45,843 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3ee1f0c896d04248be9a5dba280976fe as hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3ee1f0c896d04248be9a5dba280976fe 2024-11-10T11:03:45,847 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3ee1f0c896d04248be9a5dba280976fe, entries=3, sequenceid=29, filesize=5.2 K 2024-11-10T11:03:45,847 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e8c7241d3f64eb9b913f4c29d5167c4 as hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e8c7241d3f64eb9b913f4c29d5167c4 2024-11-10T11:03:45,851 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e8c7241d3f64eb9b913f4c29d5167c4, entries=1, sequenceid=29, filesize=5.0 K 2024-11-10T11:03:45,852 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/4a74d21257e14f5da7dfd0e16e7ac817 as hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4a74d21257e14f5da7dfd0e16e7ac817 2024-11-10T11:03:45,855 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44203/user/jenkins/test-data/86442e82-6e1a-0f3b-cf7e-ddb476ef7b6b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/4a74d21257e14f5da7dfd0e16e7ac817, entries=1, sequenceid=29, filesize=4.9 K 2024-11-10T11:03:45,856 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=29, compaction requested=false 2024-11-10T11:03:45,858 INFO [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T11:03:45,858 DEBUG [M:0;eccb6f743e07:45435 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731236625744Disabling compacts and flushes for region at 1731236625744Disabling writes for close at 1731236625744Obtaining lock to block concurrent updates at 1731236625744Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731236625744Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1731236625745 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731236625745Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731236625745Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731236625760 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731236625760Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731236625770 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731236625784 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731236625784Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731236625794 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731236625807 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731236625807Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731236625815 (+8 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731236625829 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731236625829Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb979e5: reopening flushed file at 1731236625838 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6de013f1: reopening flushed file at 1731236625843 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@667b0ab8: reopening flushed file at 1731236625847 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1490668d: reopening flushed file at 1731236625851 (+4 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 112ms, sequenceid=29, compaction requested=false at 1731236625856 (+5 ms)Writing region close event to WAL at 1731236625858 (+2 ms)Closed at 1731236625858 2024-11-10T11:03:45,858 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,858 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,859 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,859 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,859 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-10T11:03:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40079 is added to blk_1073741830_1006 (size=10311) 2024-11-10T11:03:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46667 is added to blk_1073741830_1006 (size=10311) 2024-11-10T11:03:45,861 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T11:03:45,861 INFO [M:0;eccb6f743e07:45435 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T11:03:45,861 INFO [M:0;eccb6f743e07:45435 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45435 2024-11-10T11:03:45,861 INFO [M:0;eccb6f743e07:45435 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T11:03:45,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:45,963 INFO [M:0;eccb6f743e07:45435 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T11:03:45,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45435-0x101018c63030000, quorum=127.0.0.1:62880, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T11:03:45,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@10e33487{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:03:45,966 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@814e400{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:03:45,966 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:03:45,966 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@735ef7ff{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:03:45,966 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7a47ab0e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir/,STOPPED} 2024-11-10T11:03:45,967 WARN [BP-1022027770-172.17.0.2-1731236623853 heartbeating to localhost/127.0.0.1:44203 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:03:45,967 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:03:45,967 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:03:45,967 WARN [BP-1022027770-172.17.0.2-1731236623853 heartbeating to localhost/127.0.0.1:44203 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1022027770-172.17.0.2-1731236623853 (Datanode Uuid 82b27ece-91e3-480c-aa28-1ae66fb1ad98) service to localhost/127.0.0.1:44203 2024-11-10T11:03:45,968 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data3/current/BP-1022027770-172.17.0.2-1731236623853 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:45,968 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data4/current/BP-1022027770-172.17.0.2-1731236623853 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:45,968 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:03:45,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@186dc28{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T11:03:45,970 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c8a9439{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:03:45,970 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:03:45,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1bfccdd{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:03:45,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d72fbcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir/,STOPPED} 2024-11-10T11:03:45,972 WARN [BP-1022027770-172.17.0.2-1731236623853 heartbeating to localhost/127.0.0.1:44203 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T11:03:45,972 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T11:03:45,972 WARN [BP-1022027770-172.17.0.2-1731236623853 heartbeating to localhost/127.0.0.1:44203 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1022027770-172.17.0.2-1731236623853 (Datanode Uuid f7d019af-6efb-4f36-95ff-15bf0fb9f7a0) service to localhost/127.0.0.1:44203 2024-11-10T11:03:45,972 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T11:03:45,972 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data1/current/BP-1022027770-172.17.0.2-1731236623853 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:45,972 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/cluster_4f079285-2d84-6168-0ddf-8e7f6ff72dc7/data/data2/current/BP-1022027770-172.17.0.2-1731236623853 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T11:03:45,973 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T11:03:45,978 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30964c14{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T11:03:45,978 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@32d01bcd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T11:03:45,978 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T11:03:45,978 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2a7c9e07{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T11:03:45,978 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@46dfb375{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/40885d55-7fca-950c-1a30-04c3b81fbd04/hadoop.log.dir/,STOPPED} 2024-11-10T11:03:45,984 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T11:03:45,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T11:03:46,007 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=270 (was 232) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: globalEventExecutor-1-20 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113) app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44203 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44203 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44203 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44203 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44203 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44203 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:44203 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44203 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=535 (was 506) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=29 (was 23) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8592 (was 8605)